From a468f11b664f85ee60ebb398e83db0fef4f0a751 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Tue, 11 Aug 2026 16:36:58 +0900 Subject: [PATCH 01/15] Revert "fix: preserve broadcast build-side fetch (#618)": see https://github.com/datafusion-contrib/datafusion-distributed/pull/618#issuecomment-5250278429 This reverts commit 2dc7a369c96433584622bd9809a1022b2a0907e8. --- src/distributed_planner/insert_broadcast.rs | 8 ++++---- tests/multi_task_collect_join_repros.rs | 14 -------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/distributed_planner/insert_broadcast.rs b/src/distributed_planner/insert_broadcast.rs index 6986223b1..64e9a3e41 100644 --- a/src/distributed_planner/insert_broadcast.rs +++ b/src/distributed_planner/insert_broadcast.rs @@ -133,17 +133,17 @@ pub(super) fn insert_broadcast_execs( return Ok(Transformed::no(node)); }; - let (broadcast_input, coalesce_fetch) = build_child + let broadcast_input = build_child .downcast_ref::() .map_or_else( - || (Arc::clone(build_child), None), - |coalesce| (Arc::clone(coalesce.input()), coalesce.fetch()), + || Arc::clone(build_child), + |coalesce| Arc::clone(coalesce.input()), ); // consumer_task_count=1 is a placeholder and will be corrected during optimizer rule. let broadcast: Arc = Arc::new(BroadcastExec::new(broadcast_input, 1)); let new_build_child: Arc = - Arc::new(CoalescePartitionsExec::new(broadcast).with_fetch(coalesce_fetch)); + Arc::new(CoalescePartitionsExec::new(broadcast)); let mut new_children: Vec> = children.into_iter().cloned().collect(); new_children[0] = new_build_child; diff --git a/tests/multi_task_collect_join_repros.rs b/tests/multi_task_collect_join_repros.rs index 91e0d5dcf..46e01791f 100644 --- a/tests/multi_task_collect_join_repros.rs +++ b/tests/multi_task_collect_join_repros.rs @@ -343,20 +343,6 @@ mod tests { .unwrap(); } - /// A build-side `LIMIT` is carried by the `CoalescePartitionsExec` that a - /// broadcast rewrite replaces. The replacement must preserve that fetch or - /// the join observes every build row instead of the requested 50. - #[tokio::test] - async fn build_side_fetch_is_preserved_by_broadcast() { - assert_distributed_matches_single_node( - "SELECT count(*) FROM (SELECT id FROM build_side LIMIT 50) b \ - JOIN probe_side p ON b.id = p.id", - true, - ) - .await - .unwrap(); - } - fn data_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("target/multi_task_collect_join_repros") } From 6fff7bd16c26b36e66f2d9ec121cbf8660db2372 Mon Sep 17 00:00:00 2001 From: RJ Barman Date: Mon, 27 Jul 2026 15:28:25 -0600 Subject: [PATCH 02/15] Upgrade to pre-release DataFusion 55 (#44) As part of [this](https://github.com/paradedb/paradedb/issues/5553) switch to pre-release DataFusion 55. This PR updates the DF rev used, fixes the compile errors, updates a bunch of test outputs, and removes the unnecessary statistics wrapper. --------- Co-authored-by: Mohammad Dashti --- Cargo.lock | 356 +- Cargo.toml | 22 +- benchmarks/Cargo.toml | 12 +- benchmarks/src/datasets/tpch.rs | 6 +- cli/Cargo.toml | 2 +- cli/src/main.rs | 8 +- console/Cargo.toml | 2 +- console/src/main.rs | 8 +- console/src/worker.rs | 50 +- examples/custom_execution_plan.rs | 8 +- examples/custom_worker_url_routing.rs | 11 +- examples/work_unit_feed.rs | 8 +- src/codec/distributed_codec.rs | 83 +- src/coordinator/prepare_dynamic_plan.rs | 5 +- src/distributed_ext.rs | 6 +- .../distributed_query_planner.rs | 96 +- .../inject_network_boundaries.rs | 10 +- src/distributed_planner/insert_broadcast.rs | 24 +- .../normalize_collect_joins.rs | 2 +- .../statistics/complexity_cpu.rs | 23 +- .../statistics/complexity_memory.rs | 8 +- .../statistics/plan_statistics.rs | 91 +- src/events/defaults/file_scan_config.rs | 58 +- src/execution_plans/common.rs | 10 + src/execution_plans/distributed_leaf.rs | 13 +- src/protocol/grpc/generated/worker.rs | 9 +- src/protocol/grpc/metrics_proto.rs | 27 + src/protocol/grpc/worker.proto | 7 +- src/stage.rs | 8 +- src/test_utils/routing.rs | 13 +- src/test_utils/test_work_unit_feed.rs | 10 +- src/test_utils/work_unit_file_scan.rs | 33 +- src/worker/session_builder.rs | 6 +- tests/clickbench_plans_test.rs | 221 +- tests/custom_config_extension.rs | 6 +- tests/custom_extension_codec.rs | 6 +- tests/distributed_aggregation.rs | 10 +- tests/distributed_unions.rs | 71 +- tests/error_propagation.rs | 6 +- tests/join.rs | 28 +- tests/multi_task_collect_join_repros.rs | 46 +- tests/stateful_execution_plan.rs | 2 + tests/task_estimator_test.rs | 8 +- tests/tpcds_plans_test.rs | 9812 ++++++++--------- tests/tpch_plans_test.rs | 677 +- tests/udfs.rs | 4 +- tests/work_unit_feed.rs | 14 +- 47 files changed, 5847 insertions(+), 6099 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b5e39abb..08320584b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,7 +132,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -143,7 +143,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -170,6 +170,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -184,9 +193,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" dependencies = [ "arrow-arith", "arrow-array", @@ -207,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" dependencies = [ "arrow-array", "arrow-buffer", @@ -221,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" dependencies = [ "ahash", "arrow-buffer", @@ -240,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" dependencies = [ "bytes", "half", @@ -276,9 +285,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" dependencies = [ "arrow-array", "arrow-buffer", @@ -298,9 +307,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" dependencies = [ "arrow-array", "arrow-cast", @@ -313,9 +322,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" dependencies = [ "arrow-buffer", "arrow-schema", @@ -326,13 +335,14 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" +checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", + "arrow-data", "arrow-ipc", "arrow-schema", "base64", @@ -346,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" dependencies = [ "arrow-array", "arrow-buffer", @@ -362,9 +372,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" dependencies = [ "arrow-array", "arrow-buffer", @@ -387,9 +397,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" dependencies = [ "arrow-array", "arrow-buffer", @@ -400,9 +410,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" dependencies = [ "arrow-array", "arrow-buffer", @@ -413,9 +423,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" dependencies = [ "serde_core", "serde_json", @@ -423,9 +433,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" dependencies = [ "ahash", "arrow-array", @@ -437,9 +447,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" dependencies = [ "arrow-array", "arrow-buffer", @@ -529,9 +539,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.17" +version = "1.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "517aa062d8bd9015ee23d6daa5e1c1372328412fdae4e6c4c1be9b69c6ad37a2" +checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" dependencies = [ "aws-credential-types", "aws-runtime", @@ -619,10 +629,11 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.99.0" +version = "1.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f4055e6099b2ec264abdc0d9bbfffce306c1601809275c861594779a0b04b45" +checksum = "b647baea49ff551960b904f905681e9b4765a6c4ea08631e89dc52d8bd3f5896" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -643,10 +654,11 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.101.0" +version = "1.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f009ba0284c5d696425fd7b4dcc5b189f5726f4041b7a5794daecb3a68d598" +checksum = "7ae401c65ff288aa7873117fe535cd32b7b1bb0bc43751d28901a1d5f20636b9" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -667,10 +679,11 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.104.0" +version = "1.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa6622798e19e6a76b690562085dd4771c736cd48343464a53ab4ae2f2c9f84" +checksum = "4c80de7bb7d03e9ca8c9fd7b489f20f3948d3f3be91a7953591347d238115408" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -770,9 +783,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.6" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "517089205f18ab4adc5a3e02888cb139bbbbb2e168eac9f396216925d1fbeaf5" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -826,9 +839,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -866,9 +879,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.4.8" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" dependencies = [ "base64-simd", "bytes", @@ -1118,12 +1131,6 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -1201,9 +1208,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1774,8 +1781,7 @@ dependencies = [ [[package]] name = "datafusion" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-schema", @@ -1811,7 +1817,7 @@ dependencies = [ "flate2", "futures", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1828,8 +1834,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -1843,7 +1848,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1853,8 +1858,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -1868,16 +1872,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-cli" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a9533845ab888508718c2ad540cff205d1111525a9952def8b0e584b6453784" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -1904,8 +1908,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-ipc", @@ -1916,9 +1919,10 @@ dependencies = [ "hashbrown 0.17.1", "hex", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -1931,8 +1935,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "futures", "log", @@ -1942,8 +1945,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-compression", @@ -1963,7 +1965,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools 0.14.0", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1978,8 +1980,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-ipc", @@ -1994,7 +1995,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "object_store", "tokio", ] @@ -2002,8 +2003,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb517d08967d536284ce70afb5fe8583133779249f2d7b90587d339741a7f195" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-avro", @@ -2021,8 +2021,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -2044,8 +2043,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -2067,10 +2065,10 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -2087,7 +2085,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools 0.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -2202,18 +2200,17 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" [[package]] name = "datafusion-execution" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -2223,16 +2220,18 @@ dependencies = [ "object_store", "parking_lot", "parquet", + "pin-project-lite", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-schema", @@ -2245,7 +2244,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -2254,20 +2253,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-buffer", @@ -2284,7 +2281,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools 0.14.0", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -2298,8 +2295,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2310,7 +2306,6 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", "log", "num-traits", @@ -2319,8 +2314,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2331,8 +2325,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-ord", @@ -2347,7 +2340,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools 0.14.0", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -2356,8 +2349,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "async-trait", @@ -2372,8 +2364,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2389,8 +2380,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2399,8 +2389,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "datafusion-doc", "quote", @@ -2410,8 +2399,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "chrono", @@ -2420,7 +2408,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "log", "recursive", "regex", @@ -2430,8 +2418,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2439,10 +2426,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -2452,8 +2440,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2461,22 +2448,22 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools 0.14.0", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "parking_lot", "pin-project", ] @@ -2484,8 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2496,15 +2482,14 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools 0.14.0", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "arrow-data", @@ -2512,6 +2497,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -2521,23 +2507,25 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap", - "itertools 0.14.0", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "chrono", @@ -2556,6 +2544,7 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] @@ -2563,19 +2552,26 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "54.0.0" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +dependencies = [ + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "datafusion-common", @@ -2590,8 +2586,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "async-trait", "datafusion-common", @@ -2604,8 +2599,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "54.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" dependencies = [ "arrow", "bigdecimal", @@ -2618,6 +2612,7 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -2732,7 +2727,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -2824,7 +2819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3664,12 +3659,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.0" @@ -3684,7 +3673,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3711,6 +3700,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -4043,9 +4041,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memmem" @@ -4374,15 +4372,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-float" version = "4.6.0" @@ -4435,9 +4424,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" dependencies = [ "ahash", "arrow-array", @@ -4464,7 +4453,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -5262,7 +5250,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5442,6 +5430,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -5598,7 +5587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5639,7 +5628,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5817,7 +5806,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5862,7 +5851,7 @@ dependencies = [ "nix 0.29.0", "num-derive", "num-traits", - "ordered-float 4.6.0", + "ordered-float", "pest", "pest_derive", "phf 0.11.3", @@ -5974,17 +5963,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float 2.10.1", -] - [[package]] name = "time" version = "0.3.47" @@ -6225,13 +6203,13 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpchgen" -version = "2.0.2" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=438e9c2dbc25b2fff82c0efc08b3f13b5707874f#438e9c2dbc25b2fff82c0efc08b3f13b5707874f" +version = "3.0.0" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=0f6747e592dbdf9dadd5cd3c5a43e867267bb62f#0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" [[package]] name = "tpchgen-arrow" -version = "2.0.2" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=438e9c2dbc25b2fff82c0efc08b3f13b5707874f#438e9c2dbc25b2fff82c0efc08b3f13b5707874f" +version = "3.0.0" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=0f6747e592dbdf9dadd5cd3c5a43e867267bb62f#0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" dependencies = [ "arrow", "tpchgen", @@ -6649,7 +6627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" dependencies = [ "log", - "ordered-float 4.6.0", + "ordered-float", "strsim 0.11.1", "thiserror 1.0.69", "wezterm-dynamic-derive", @@ -6701,7 +6679,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 99262de04..27843802e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,8 @@ members = [ ] [workspace.dependencies] -datafusion = { version = "54.0.0", default-features = false } -datafusion-proto = { version = "54.0.0", default-features = false } +datafusion = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } +datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } [package] name = "datafusion-distributed" @@ -19,7 +19,7 @@ documentation = "https://datafusion-contrib.github.io/datafusion-distributed/" repository = "https://github.com/datafusion-contrib/datafusion-distributed" [dependencies] -chrono = { version = "0.4.44" } +chrono = { version = "0.4.45" } datafusion = { workspace = true, features = [ "unicode_expressions", "datetime_expressions", @@ -50,16 +50,16 @@ bincode = "1" tonic-prost = "0.14.2" # grpc-specific features -arrow-flight = { version = "58", optional = true } -arrow-select = { version = "58", optional = true } -arrow-ipc = { version = "58", features = ["zstd"], optional = true } +arrow-flight = { version = "59", optional = true } +arrow-select = { version = "59", optional = true } +arrow-ipc = { version = "59", features = ["zstd"], optional = true } tonic = { version = "0.14.1", features = ["transport"], optional = true } tower = { version = "0.5.2", optional = true } # integration_tests deps insta = { version = "1.46.0", features = ["filters"], optional = true } -parquet = { version = "58", optional = true } -arrow = { version = "58", optional = true, features = ["test_utils"] } +parquet = { version = "59", optional = true } +arrow = { version = "59", optional = true, features = ["test_utils"] } hyper-util = { version = "0.1.16", optional = true } [features] @@ -76,7 +76,7 @@ grpc = [ "arrow-select", "arrow-ipc", "tonic", - "tower" + "tower", ] integration = ["insta", "dep:parquet", "arrow", "hyper-util", "grpc", "parquet", "sql"] @@ -98,8 +98,8 @@ datafusion = { workspace = true, features = ["parquet", "sql"] } datafusion-proto = { workspace = true, features = ["parquet"] } structopt = "0.3" insta = { version = "1.46.0", features = ["filters"] } -parquet = "58" -arrow = { version = "58", features = ["test_utils"] } +parquet = "59" +arrow = { version = "59", features = ["test_utils"] } tokio-stream = { version = "0.1.17", features = ["sync"] } hyper-util = "0.1.16" pretty_assertions = "1.4" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 19f64499d..13f795087 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -7,21 +7,21 @@ default-run = "dfbench" [dependencies] datafusion = { workspace = true } datafusion-distributed = { path = "..", features = [ - "integration", - "system-metrics", + "integration", + "system-metrics", ] } tokio = { version = "1.48", features = ["full"] } -parquet = { version = "58" } +parquet = { version = "59" } structopt = { version = "0.3.26" } log = "0.4.27" serde = "1.0.219" serde_json = "1.0.141" env_logger = "0.11.8" futures = "0.3.31" -arrow = { version = "58", features = ["test_utils"] } +arrow = { version = "59", features = ["test_utils"] } tonic = { version = "0.14.1", features = ["transport"] } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "0f6747e592dbdf9dadd5cd3c5a43e867267bb62f" } reqwest = "0.12" zip = "6.0" sketches-ddsketch = "0.3" diff --git a/benchmarks/src/datasets/tpch.rs b/benchmarks/src/datasets/tpch.rs index 417fcbdab..0ae618171 100644 --- a/benchmarks/src/datasets/tpch.rs +++ b/benchmarks/src/datasets/tpch.rs @@ -1,4 +1,5 @@ use super::common; +use arrow::error::ArrowError; use arrow::record_batch::RecordBatch; use datafusion::error::DataFusionError; use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties}; @@ -27,11 +28,12 @@ fn generate_table( data_dir: &Path, ) -> Result<(), Box> where - A: Iterator, + A: Iterator>, { let output_path = data_dir.join(format!("{table_name}.parquet")); if let Some(first_batch) = data_source.next() { + let first_batch = first_batch?; let file = fs::File::create(&output_path)?; let props = WriterProperties::builder().build(); let mut writer = ArrowWriter::try_new(file, first_batch.schema(), Some(props))?; @@ -39,7 +41,7 @@ where writer.write(&first_batch)?; for batch in data_source { - writer.write(&batch)?; + writer.write(&batch?)?; } writer.close()?; diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 66c855659..fb2d4430d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,7 +6,7 @@ edition = "2024" [dependencies] datafusion = { workspace = true } datafusion-distributed = { path = "..", features = ["avro", "integration"] } -datafusion-cli = { version = "54", default-features = false } +datafusion-cli = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } tokio = { version = "1.48", features = ["full"] } clap = { version = "4", features = ["derive"] } env_logger = "0.11" diff --git a/cli/src/main.rs b/cli/src/main.rs index f3a10491e..260824569 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -18,8 +18,7 @@ // File mainly copied from https://github.com/apache/datafusion/blob/main/datafusion-cli/src/main.rs use clap::Parser; -use datafusion::common::config_err; -use datafusion::config::ConfigOptions; +use datafusion::config::{ConfigNonZeroUsize, ConfigOptions}; use datafusion::error::{DataFusionError, Result}; use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionConfig; @@ -211,10 +210,7 @@ fn get_session_config(args: &Args) -> Result { let mut config_options = ConfigOptions::from_env()?; if let Some(batch_size) = args.batch_size { - if batch_size == 0 { - return config_err!("batch_size must be greater than 0"); - } - config_options.execution.batch_size = batch_size; + config_options.execution.batch_size = ConfigNonZeroUsize::try_new(batch_size)?; }; // use easier to understand "tree" mode by default diff --git a/console/Cargo.toml b/console/Cargo.toml index 5267842e2..5f0433752 100644 --- a/console/Cargo.toml +++ b/console/Cargo.toml @@ -17,7 +17,7 @@ url = "2.5.7" tokio-stream = "0.1.18" [dev-dependencies] -arrow = "58" +arrow = "59" async-trait = "0.1.89" datafusion-distributed-benchmarks = { path = "../benchmarks" } futures = "0.3.31" diff --git a/console/src/main.rs b/console/src/main.rs index 731b504ba..ab1171d78 100644 --- a/console/src/main.rs +++ b/console/src/main.rs @@ -63,10 +63,10 @@ async fn run_app( terminal.draw(|frame| ui::render(frame, app))?; // Check for keyboard input (16ms timeout ~ 60fps responsiveness) - if event::poll(Duration::from_millis(16))? { - if let Event::Key(key) = event::read()? { - input::handle_key_event(app, key); - } + if event::poll(Duration::from_millis(16))? + && let Event::Key(key) = event::read()? + { + input::handle_key_event(app, key); } if app.should_quit { diff --git a/console/src/worker.rs b/console/src/worker.rs index 4a23c5b90..44c062607 100644 --- a/console/src/worker.rs +++ b/console/src/worker.rs @@ -176,32 +176,32 @@ impl WorkerConn { // Detect completed tasks: tasks that were running but disappeared for old_task in &self.tasks { - if old_task.status == TaskStatus::Running as i32 { - if let Some(sk) = &old_task.task_key { - let key = (sk.query_id.clone(), sk.stage_id, sk.task_number); - if !new_task_keys.contains(&key) { - // Task disappeared — assume completed - let observed_duration = self - .task_first_seen - .get(&key) - .map(|first| first.elapsed()) - .unwrap_or_default(); - - self.completed_tasks.push_front(CompletedTaskRecord { - query_id: sk.query_id.clone(), - stage_id: sk.stage_id, - task_number: sk.task_number, - observed_duration, - }); - - // Maintain bounded size - while self.completed_tasks.len() > MAX_COMPLETED_TASKS { - self.completed_tasks.pop_back(); - } - - // Remove from first_seen tracking - self.task_first_seen.remove(&key); + if old_task.status == TaskStatus::Running as i32 + && let Some(sk) = &old_task.task_key + { + let key = (sk.query_id.clone(), sk.stage_id, sk.task_number); + if !new_task_keys.contains(&key) { + // Task disappeared — assume completed + let observed_duration = self + .task_first_seen + .get(&key) + .map(|first| first.elapsed()) + .unwrap_or_default(); + + self.completed_tasks.push_front(CompletedTaskRecord { + query_id: sk.query_id.clone(), + stage_id: sk.stage_id, + task_number: sk.task_number, + observed_duration, + }); + + // Maintain bounded size + while self.completed_tasks.len() > MAX_COMPLETED_TASKS { + self.completed_tasks.pop_back(); } + + // Remove from first_seen tracking + self.task_first_seen.remove(&key); } } } diff --git a/examples/custom_execution_plan.rs b/examples/custom_execution_plan.rs index 3c1636545..61ae45dfb 100644 --- a/examples/custom_execution_plan.rs +++ b/examples/custom_execution_plan.rs @@ -242,6 +242,7 @@ impl PhysicalExtensionCodec for NumbersExecCodec { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, ) -> Result> { if !inputs.is_empty() { return internal_err!("NumbersExec should have no children, got {}", inputs.len()); @@ -262,7 +263,12 @@ impl PhysicalExtensionCodec for NumbersExecCodec { ))) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, + ) -> Result<()> { let Some(exec) = node.downcast_ref::() else { return internal_err!("Expected plan to be NumbersExec, but was {}", node.name()); }; diff --git a/examples/custom_worker_url_routing.rs b/examples/custom_worker_url_routing.rs index 12724bb31..62f2e9740 100644 --- a/examples/custom_worker_url_routing.rs +++ b/examples/custom_worker_url_routing.rs @@ -45,6 +45,7 @@ use datafusion_distributed::{ DistributedLeafExec, RouteTasksEvent, RouteTasksEventResponse, ScaleUpLeafNodeEvent, ScaleUpLeafNodeEventResponse, SessionStateBuilderExt, WorkerQueryContext, display_plan_ascii, }; +use datafusion_proto::TryFromProto; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf; use futures::TryStreamExt; @@ -151,7 +152,7 @@ impl ExecutionPlan for CacheExec { fn hash_key(file_group: &FileGroup) -> usize { let mut hasher = DefaultHasher::new(); for file in file_group.files() { - let serialized: protobuf::PartitionedFile = file.try_into().unwrap(); + let serialized = protobuf::PartitionedFile::try_from_proto(file).unwrap(); hasher.write(&serialized.encode_to_vec()); } hasher.finish() as usize @@ -247,6 +248,7 @@ impl PhysicalExtensionCodec for CachedFileScanCodec { _buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, ) -> Result> { let [child] = inputs else { return internal_err!("CacheExec expects exactly 1 child, got {}", inputs.len()); @@ -254,7 +256,12 @@ impl PhysicalExtensionCodec for CachedFileScanCodec { Ok(CacheExec::new(Arc::clone(child))) } - fn try_encode(&self, node: Arc, _buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + _buf: &mut Vec, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, + ) -> Result<()> { if node.downcast_ref::().is_none() { return internal_err!("Expected CacheExec, got {}", node.name()); } diff --git a/examples/work_unit_feed.rs b/examples/work_unit_feed.rs index 7b62b1914..3daa12223 100644 --- a/examples/work_unit_feed.rs +++ b/examples/work_unit_feed.rs @@ -210,6 +210,7 @@ impl PhysicalExtensionCodec for RemoteScanExecCodec { buf: &[u8], _inputs: &[Arc], _ctx: &TaskContext, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, ) -> Result> { let p = RemoteScanProto::decode(buf).map_err(|e| proto_error(format!("{e}")))?; let feed = WorkUnitFeed::::from_proto( @@ -224,7 +225,12 @@ impl PhysicalExtensionCodec for RemoteScanExecCodec { ))) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, + ) -> Result<()> { let exec = node .downcast_ref::() .ok_or_else(|| proto_error(format!("expected RemoteScanExec, got {}", node.name())))?; diff --git a/src/codec/distributed_codec.rs b/src/codec/distributed_codec.rs index 1ea275414..315b00618 100644 --- a/src/codec/distributed_codec.rs +++ b/src/codec/distributed_codec.rs @@ -22,8 +22,8 @@ use datafusion::prelude::SessionConfig; use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; use datafusion_proto::physical_plan::to_proto::serialize_partitioning; use datafusion_proto::physical_plan::{ - ComposedPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec, - PhysicalPlanDecodeContext, + ComposedPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalPlanDecodeContext, + PhysicalProtoConverterExtension, }; use datafusion_proto::protobuf; use datafusion_proto::protobuf::proto_error; @@ -51,6 +51,7 @@ impl PhysicalExtensionCodec for DistributedCodec { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result> { let DistributedExecProto { node: Some(distributed_exec_node), @@ -107,12 +108,12 @@ impl PhysicalExtensionCodec for DistributedCodec { .map(|s| s.try_into()) .ok_or(proto_error("NetworkShuffleExec is missing schema"))??; - let decode_ctx = PhysicalPlanDecodeContext::new(ctx, &DistributedCodec {}); + let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self); let partitioning = parse_protobuf_partitioning( partitioning.as_ref(), &decode_ctx, &schema, - &DefaultPhysicalProtoConverter {}, + proto_converter, )? .ok_or(proto_error("NetworkShuffleExec is missing partitioning"))?; @@ -137,7 +138,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning.as_ref(), &decode_ctx, &schema, - &DefaultPhysicalProtoConverter {}, + proto_converter, )? .ok_or(proto_error("NetworkCoalesceExec is missing partitioning"))?; @@ -162,7 +163,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning.as_ref(), &decode_ctx, &schema, - &DefaultPhysicalProtoConverter {}, + proto_converter, )? .ok_or(proto_error("NetworkBroadcastExec is missing partitioning"))?; @@ -241,7 +242,12 @@ impl PhysicalExtensionCodec for DistributedCodec { } } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { fn encode_stage_proto(stage: &Stage) -> Result { Ok(match stage { Stage::Local(local) => StageProto { @@ -271,7 +277,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning: Some(serialize_partitioning( node.properties().output_partitioning(), &DistributedCodec {}, - &DefaultPhysicalProtoConverter {}, + proto_converter, )?), input_stage: Some(encode_stage_proto(node.input_stage())?), }; @@ -287,7 +293,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning: Some(serialize_partitioning( node.properties().output_partitioning(), &DistributedCodec {}, - &DefaultPhysicalProtoConverter {}, + proto_converter, )?), input_stage: Some(encode_stage_proto(node.input_stage())?), }; @@ -303,7 +309,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning: Some(serialize_partitioning( node.properties().output_partitioning(), &DistributedCodec {}, - &DefaultPhysicalProtoConverter {}, + proto_converter, )?), input_stage: Some(encode_stage_proto(node.input_stage())?), }; @@ -555,6 +561,7 @@ mod tests { physical_expr::{Partitioning, PhysicalSortExpr, expressions::Column, expressions::col}, physical_plan::{ExecutionPlan, displayable, sorts::sort::SortExec, union::UnionExec}, }; + use datafusion_proto::physical_plan::DefaultPhysicalProtoConverter; fn empty_exec() -> Arc { Arc::new(EmptyExec::new(SchemaRef::new(Schema::empty()))) @@ -602,9 +609,9 @@ mod tests { Arc::new(new_network_hash_shuffle_exec(part, schema, dummy_stage())); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[], &ctx)?; + let decoded = codec.try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -632,9 +639,9 @@ mod tests { Arc::new(NetworkCoalesceExec::try_new(union.clone(), 1, 1)?); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[union], &ctx)?; + let decoded = codec.try_decode(&buf, &[union], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -665,9 +672,9 @@ mod tests { Arc::new(NetworkCoalesceExec::try_new(sort.clone(), 1, 1)?); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[sort], &ctx)?; + let decoded = codec.try_decode(&buf, &[sort], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -686,9 +693,9 @@ mod tests { )); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[], &ctx)?; + let decoded = codec.try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -708,9 +715,14 @@ mod tests { )); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; - - let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; + + let decoded = codec.try_decode( + &buf, + &[empty_exec()], + &ctx, + &DefaultPhysicalProtoConverter {}, + )?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -729,9 +741,14 @@ mod tests { )); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; - - let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; + + let decoded = codec.try_decode( + &buf, + &[empty_exec()], + &ctx, + &DefaultPhysicalProtoConverter {}, + )?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -753,9 +770,9 @@ mod tests { Arc::new(NetworkCoalesceExec::try_new(flight.clone(), 1, 1)?); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[flight], &ctx)?; + let decoded = codec.try_decode(&buf, &[flight], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -783,9 +800,9 @@ mod tests { Arc::new(NetworkCoalesceExec::try_new(union.clone(), 1, 1)?); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; - let decoded = codec.try_decode(&buf, &[union], &ctx)?; + let decoded = codec.try_decode(&buf, &[union], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -816,9 +833,15 @@ mod tests { )?); let mut buf = Vec::new(); - codec.try_encode(plan.clone(), &mut buf)?; + codec.try_encode(plan.clone(), &mut buf, &DefaultPhysicalProtoConverter {})?; + + let decoded = codec.try_decode( + &buf, + &[left, right], + &ctx, + &DefaultPhysicalProtoConverter {}, + )?; - let decoded = codec.try_decode(&buf, &[left, right], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) diff --git a/src/coordinator/prepare_dynamic_plan.rs b/src/coordinator/prepare_dynamic_plan.rs index 101a6c960..39301ce81 100644 --- a/src/coordinator/prepare_dynamic_plan.rs +++ b/src/coordinator/prepare_dynamic_plan.rs @@ -17,7 +17,8 @@ use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::common::{Result, exec_err, plan_err}; use datafusion::physical_plan::metrics::MetricsSet; use datafusion::physical_plan::{ - ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Statistics, + ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, Statistics, StatisticsArgs, + StatisticsContext, }; use futures::{Stream, StreamExt}; use std::any::TypeId; @@ -336,7 +337,7 @@ fn estimated_driver_path_leaf_rows(plan: &Arc) -> Option], ctx: &TaskContext) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext, ext: &dyn PhysicalProtoConverterExtension) -> datafusion::common::Result> { /// todo!() /// } /// - /// fn try_encode(&self, node: Arc, buf: &mut Vec) -> datafusion::common::Result<()> { + /// fn try_encode(&self, node: Arc, buf: &mut Vec, ext: &dyn PhysicalProtoConverterExtension) -> datafusion::common::Result<()> { /// todo!() /// } /// } diff --git a/src/distributed_planner/distributed_query_planner.rs b/src/distributed_planner/distributed_query_planner.rs index fd5fdfc5e..faf5f447c 100644 --- a/src/distributed_planner/distributed_query_planner.rs +++ b/src/distributed_planner/distributed_query_planner.rs @@ -222,14 +222,14 @@ mod tests { let physical_plan_ascii = TestPlanBuilder::default() .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=4 - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── @@ -253,14 +253,14 @@ mod tests { .num_workers(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=4 - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 └────────────────────────────────────────────────── @@ -283,10 +283,10 @@ mod tests { .num_workers(0) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=3 AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] @@ -306,8 +306,8 @@ mod tests { assert_snapshot!(physical_plan_ascii, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── @@ -331,10 +331,10 @@ mod tests { .distributed_file_scan_config_bytes_per_partition(128 * 1024 * 1024) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=3 AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] @@ -350,14 +350,14 @@ mod tests { let physical_plan_ascii = TestPlanBuilder::default() .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=4 - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 └────────────────────────────────────────────────── @@ -400,9 +400,9 @@ mod tests { ┌───── Stage 2 ── tasks=3, partitions=12 │ RepartitionExec: partitioning=Hash([RainToday@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); } @@ -473,9 +473,9 @@ mod tests { │ FilterExec: RainToday@1 = no, projection=[MaxTemp@0, RainTomorrow@2] │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] └────────────────────────────────────────────────── "); } @@ -599,7 +599,7 @@ mod tests { │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={1 group: [[/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -615,17 +615,17 @@ mod tests { .distributed_max_tasks_per_stage(2) .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" ┌───── DistributedExec │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=2, partitions=6 + ┌───── Stage 1 ── tasks=2, partitions=8 │ DistributedUnionExec: t0:[c0] t1:[c1] │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={1 group: [[/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..], [/testdata/flights-1m.parquet:..]]}, projection=[1 as Int64(1)], file_type=parquet └────────────────────────────────────────────────── "); } @@ -805,9 +805,9 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -841,9 +841,9 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -851,9 +851,9 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -875,11 +875,11 @@ mod tests { let physical_plan_ascii = TestPlanBuilder::default() .physical_plan_as_ascii(query, false) .await; - assert_snapshot!(physical_plan_ascii, @r" + assert_snapshot!(physical_plan_ascii, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); let physical_plan_ascii = TestPlanBuilder::default() @@ -896,9 +896,9 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=27 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 @@ -940,17 +940,17 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=3, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 @@ -998,9 +998,9 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=1, partitions=9 │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 diff --git a/src/distributed_planner/inject_network_boundaries.rs b/src/distributed_planner/inject_network_boundaries.rs index 7a7f721bf..f0194de35 100644 --- a/src/distributed_planner/inject_network_boundaries.rs +++ b/src/distributed_planner/inject_network_boundaries.rs @@ -721,11 +721,11 @@ mod tests { .distributed_planner(false) .broadcast_joins(false); let annotated = annotate_test_plan(test_plan_builder, query).await; - assert_snapshot!(annotated, @r" + assert_snapshot!(annotated, @" SortPreservingMergeExec: task_count=Maximum(1) NetworkCoalesceExec: task_count=Maximum(1) - SortExec: task_count=Desired(3) - ProjectionExec: task_count=Desired(3) + ProjectionExec: task_count=Desired(3) + SortExec: task_count=Desired(3) AggregateExec: task_count=Desired(3) NetworkShuffleExec: task_count=Desired(3) RepartitionExec: task_count=Desired(4) @@ -1047,10 +1047,10 @@ mod tests { .await .physical_plan_as_string(query) .await; - assert_snapshot!(physical_plan_string, @r" + assert_snapshot!(physical_plan_string, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); // With target_partitions=1, there is no CoalescePartitionsExec initially diff --git a/src/distributed_planner/insert_broadcast.rs b/src/distributed_planner/insert_broadcast.rs index 64e9a3e41..cfc89f27a 100644 --- a/src/distributed_planner/insert_broadcast.rs +++ b/src/distributed_planner/insert_broadcast.rs @@ -194,19 +194,19 @@ mod tests { .num_workers(4) .physical_plan_as_string(query) .await; - assert_snapshot!(physical_plan_string, @r" + assert_snapshot!(physical_plan_string, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); let plan = sql_to_plan_with_broadcast(query, true, 4).await; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -222,18 +222,18 @@ mod tests { .num_workers(4) .physical_plan_as_string(query) .await; - assert_snapshot!(physical_plan_string, @r" + assert_snapshot!(physical_plan_string, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); let plan = sql_to_plan_with_broadcast(query, true, 1).await; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -245,11 +245,11 @@ mod tests { ON a."RainToday" = b."RainToday" "#; let plan = sql_to_plan_with_broadcast(query, true, 4).await; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -261,11 +261,11 @@ mod tests { ON a."RainToday" = b."RainToday" "#; let plan = sql_to_plan_with_broadcast(query, false, 4).await; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] CoalescePartitionsExec DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } diff --git a/src/distributed_planner/normalize_collect_joins.rs b/src/distributed_planner/normalize_collect_joins.rs index 826c8fd48..fd4398e6a 100644 --- a/src/distributed_planner/normalize_collect_joins.rs +++ b/src/distributed_planner/normalize_collect_joins.rs @@ -231,7 +231,7 @@ mod tests { RepartitionExec: partitioning=Hash([RainToday@1], 3), input_partitions=3 DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet RepartitionExec: partitioning=Hash([RainToday@1], 3), input_partitions=3 - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } diff --git a/src/distributed_planner/statistics/complexity_cpu.rs b/src/distributed_planner/statistics/complexity_cpu.rs index 5d6cad348..abf2d4579 100644 --- a/src/distributed_planner/statistics/complexity_cpu.rs +++ b/src/distributed_planner/statistics/complexity_cpu.rs @@ -235,6 +235,13 @@ pub(super) fn complexity_cpu(node: &Arc) -> Complexity { n = n.plus(hashed_or_sorted_key_complexity(expr)) } } + // Routing a row to its range requires evaluating and comparing the ordering key, + // which costs about the same as hashing it. + Partitioning::Range(range) => { + for sort_expr in range.ordering() { + n = n.plus(hashed_or_sorted_key_complexity(&sort_expr.expr)) + } + } Partitioning::RoundRobinBatch(_) => {} Partitioning::UnknownPartitioning(_) => {} }; @@ -532,9 +539,9 @@ mod tests { .target_partitions(1) .physical_plan(r#"SELECT * FROM weather ORDER BY "WindGustDir" LIMIT 10"#) .await; - assert_snapshot!(plan_costs(topk), @r" + assert_snapshot!(plan_costs(topk), @" O((Cols+Col5)*Log(out_Cols)) | SortExec: TopK(fetch=10), expr=[WindGustDir@5 ASC NULLS LAST], preserve_partitioning=[false] - O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[WindGustDir@5 ASC NULLS LAST] + O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[WindGustDir@5 ASC NULLS LAST], dynamic_rg_pruning=eligible "); } @@ -581,10 +588,10 @@ mod tests { "#, ) .await; - assert_snapshot!(plan_costs(plan), @r" + assert_snapshot!(plan_costs(plan), @" O(((2*left_Cols)+left_Col1+right_Cols+right_Col1)) | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -605,10 +612,10 @@ mod tests { "#, ) .await; - assert_snapshot!(plan_costs(plan), @r" + assert_snapshot!(plan_costs(plan), @" O(((2*left_Cols)+left_Col1+right_Cols+right_Col1+(left_Col0+right_Col0))) | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], filter=MinTemp@0 > MaxTemp@1, projection=[MinTemp@0, MaxTemp@2] O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + O(out_Cols) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -809,12 +816,12 @@ mod tests { "#, ) .await; - assert_snapshot!(plan_costs(plan), @r" + assert_snapshot!(plan_costs(plan), @" O(((2*left_Cols)+left_Col1+right_Cols+right_Col1)) | HashJoinExec: mode=Partitioned, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] O((Cols+Col1)) | RepartitionExec: partitioning=Hash([RainToday@1], 4), input_partitions=3 O(out_Cols) | DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet O((Cols+Col1)) | RepartitionExec: partitioning=Hash([RainToday@1], 4), input_partitions=3 - O(out_Cols) | DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + O(out_Cols) | DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } diff --git a/src/distributed_planner/statistics/complexity_memory.rs b/src/distributed_planner/statistics/complexity_memory.rs index 75b2f469b..ac5b46238 100644 --- a/src/distributed_planner/statistics/complexity_memory.rs +++ b/src/distributed_planner/statistics/complexity_memory.rs @@ -116,10 +116,10 @@ mod tests { "#, ) .await; - assert_snapshot!(plan_memory(plan), @r" + assert_snapshot!(plan_memory(plan), @" M(left_Cols) | HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] M(0) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - M(0) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + M(0) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible "); } @@ -169,9 +169,9 @@ mod tests { .target_partitions(1) .physical_plan(r#"SELECT * FROM weather ORDER BY "WindGustDir" LIMIT 10"#) .await; - assert_snapshot!(plan_memory(topk), @r" + assert_snapshot!(plan_memory(topk), @" M(out_Cols) | SortExec: TopK(fetch=10), expr=[WindGustDir@5 ASC NULLS LAST], preserve_partitioning=[false] - M(0) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[WindGustDir@5 ASC NULLS LAST] + M(0) | DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[WindGustDir@5 ASC NULLS LAST], dynamic_rg_pruning=eligible "); } diff --git a/src/distributed_planner/statistics/plan_statistics.rs b/src/distributed_planner/statistics/plan_statistics.rs index d3d54b8e2..636d4b83f 100644 --- a/src/distributed_planner/statistics/plan_statistics.rs +++ b/src/distributed_planner/statistics/plan_statistics.rs @@ -1,14 +1,9 @@ use crate::distributed_planner::statistics::default_bytes_for_datatype::default_bytes_for_datatype; use datafusion::common::stats::Precision; -use datafusion::common::{Statistics, not_impl_err, plan_err}; -use datafusion::config::ConfigOptions; +use datafusion::common::{Statistics, plan_err}; use datafusion::error::Result; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_plan::execution_plan::CardinalityEffect; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use delegate::delegate; +use datafusion::physical_plan::{ExecutionPlan, StatisticsArgs}; use itertools::Itertools; -use std::fmt::Formatter; use std::sync::Arc; /// Ratio applied to the total number of rows for calculating the fallback value for NDVs. @@ -22,7 +17,10 @@ pub(super) fn plan_statistics( node: &Arc, children_stats: &[Arc], ) -> Result> { - let mut stats = partition_statistics_with_children_override(node, None, children_stats)?; + let mut stats = node + .statistics_from_inputs(children_stats, &StatisticsArgs::new())? + .as_ref() + .clone(); // If rows are absent, but the children declares rows, be conservative and assume that the node // is not going to reduce cardinality and that the row count stays the same. @@ -75,80 +73,3 @@ pub(super) fn plan_statistics( Ok(Arc::new(stats)) } - -// FIXME: because of limitations the the statistics API on DataFusion, we need to resource to -// this sketchy way of overriding child statistics, as we cannot just provide our own. -// If we don't do this: -// 1. we cannot tell nodes to compute statistics based on the ones we provide. -// 2. we recompute statistics unnecessarily across the plan -// This is tracked by https://github.com/apache/datafusion/issues/20184 upstream, and until -// that one is solved, we need to resource to this wrapper. -fn partition_statistics_with_children_override( - node: &Arc, - partition: Option, - child_stats: &[Arc], -) -> Result { - // DataFusion stats system is not very mature yet. This override layer brings in changes - // that might not have already been released or informed overrides. - let statistics_wrapped_children = child_stats - .iter() - .zip(node.children()) - .map(|(stats, child)| StatisticsWrapper { - inner: Arc::clone(child), - stats: Arc::clone(stats), - }) - .map(|v| Arc::new(v) as _) - .collect(); - - let stats = Arc::clone(node) - .with_new_children(statistics_wrapped_children)? - .partition_statistics(partition)?; - - Ok(stats.as_ref().clone()) -} - -#[derive(Debug)] -struct StatisticsWrapper { - stats: Arc, - inner: Arc, -} - -impl DisplayAs for StatisticsWrapper { - delegate! { - to self.inner { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result; - } - } -} - -impl ExecutionPlan for StatisticsWrapper { - fn partition_statistics(&self, partition: Option) -> Result> { - if partition.is_some() { - return plan_err!("StatisticsWrapper not prepared for partition-specific stats"); - } - Ok(Arc::clone(&self.stats)) - } - - delegate! { - to self.inner { - fn name(&self) -> &str; - fn properties(&self) -> &Arc; - fn maintains_input_order(&self) -> Vec; - fn benefits_from_input_partitioning(&self) -> Vec; - fn children(&self) -> Vec<&Arc>; - fn repartitioned(&self, _target_partitions: usize, _config: &ConfigOptions) -> Result>>; - fn execute(&self, partition: usize, context: Arc) -> Result; - fn supports_limit_pushdown(&self) -> bool; - fn with_fetch(&self, _limit: Option) -> Option>; - fn fetch(&self) -> Option; - fn cardinality_effect(&self) -> CardinalityEffect; - } - } - - fn with_new_children( - self: Arc, - _: Vec>, - ) -> Result> { - not_impl_err!("with_new_children not implemented") - } -} diff --git a/src/events/defaults/file_scan_config.rs b/src/events/defaults/file_scan_config.rs index 816a64792..b75ab39a3 100644 --- a/src/events/defaults/file_scan_config.rs +++ b/src/events/defaults/file_scan_config.rs @@ -5,7 +5,7 @@ use crate::events::{ }; use crate::execution_plans::DistributedLeafExec; use datafusion::catalog::memory::DataSourceExec; -use datafusion::datasource::physical_plan::{FileGroup, FileGroupPartitioner, FileScanConfig}; +use datafusion::datasource::physical_plan::{FileGroupPartitioner, FileScanConfig}; use datafusion::error::Result; use datafusion::physical_plan::ExecutionPlanProperties; use std::sync::Arc; @@ -40,26 +40,14 @@ pub(crate) fn file_scan_config_scale_up_leaf_node( let file_scan = dse.data_source().downcast_ref::()?; let partition_count = ev.plan.output_partitioning().partition_count(); - let rebalanced = if file_scan.partitioned_by_file_group { - let all_partitioned_files = file_scan - .file_groups - .iter() - .flat_map(|file_group| file_group.iter().cloned()) - .collect::>(); - rebalance_round_robin(all_partitioned_files, partition_count * ev.task_count) - .into_iter() - .map(FileGroup::new) - .collect::>() - } else { - FileGroupPartitioner::new() - .with_target_partitions(partition_count * ev.task_count) - .with_repartition_file_min_size(0) - .with_preserve_order_within_groups(!file_scan.output_ordering.is_empty()) - .repartition_file_groups(&file_scan.file_groups) - .unwrap_or_else(|| file_scan.file_groups.clone()) - .into_iter() - .collect() - }; + let rebalanced = FileGroupPartitioner::new() + .with_target_partitions(partition_count * ev.task_count) + .with_repartition_file_min_size(0) + .with_preserve_order_within_groups(!file_scan.output_ordering.is_empty()) + .repartition_file_groups(&file_scan.file_groups) + .unwrap_or_else(|| file_scan.file_groups.clone()) + .into_iter() + .collect::>(); let mut file_scan_template = file_scan.clone(); file_scan_template.file_groups.clear(); @@ -84,16 +72,6 @@ pub(crate) fn file_scan_config_scale_up_leaf_node( )))) } -fn rebalance_round_robin(items: Vec, target_groups: usize) -> Vec> { - let mut groups = (0..target_groups) - .map(|_| Vec::new()) - .collect::>>(); - for (idx, item) in items.into_iter().enumerate() { - groups[idx % target_groups].push(item); - } - groups -} - #[cfg(test)] mod tests { use super::*; @@ -156,24 +134,6 @@ mod tests { Ok(()) } - #[test] - fn test_rebalance_round_robin_fixes_group_boundary_skew() { - let groups = rebalance_round_robin((0..8).collect(), 5); - assert_eq!( - groups.iter().map(Vec::len).collect::>(), - vec![2, 2, 2, 1, 1] - ); - } - - #[test] - fn test_rebalance_round_robin_pads_with_empty_groups() { - let groups = rebalance_round_robin(vec![10, 20, 30], 5); - assert_eq!( - groups.iter().map(Vec::len).collect::>(), - vec![1, 1, 1, 0, 0] - ); - } - fn total_scan_bytes(plan: &Arc) -> usize { let dse = plan.downcast_ref::().unwrap(); let file_scan = dse.data_source().downcast_ref::().unwrap(); diff --git a/src/execution_plans/common.rs b/src/execution_plans/common.rs index 8ea0e4afb..8fb302d63 100644 --- a/src/execution_plans/common.rs +++ b/src/execution_plans/common.rs @@ -22,5 +22,15 @@ pub(super) fn scale_partitioning( Partitioning::RoundRobinBatch(p) => Partitioning::RoundRobinBatch(f(*p)), Partitioning::Hash(hash, p) => Partitioning::Hash(hash.clone(), f(*p)), Partitioning::UnknownPartitioning(p) => Partitioning::UnknownPartitioning(f(*p)), + // A range shuffle never reaches a network boundary, so its count is never + // scaled. If that changes, the caller would misroute against an unscaled + // count, so fail loud in debug rather than clone silently. + Partitioning::Range(_) => { + debug_assert!( + false, + "scale_partitioning: range partitioning is not scaled at a boundary" + ); + partitioning.clone() + } } } diff --git a/src/execution_plans/distributed_leaf.rs b/src/execution_plans/distributed_leaf.rs index 30f6b5b90..4c3bea511 100644 --- a/src/execution_plans/distributed_leaf.rs +++ b/src/execution_plans/distributed_leaf.rs @@ -2,7 +2,9 @@ use crate::DistributedTaskContext; use datafusion::common::{Result, Statistics, exec_err, not_impl_err, plan_err}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr_common::metrics::MetricsSet; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs, StatisticsContext, +}; use std::fmt::Formatter; use std::sync::Arc; @@ -191,7 +193,12 @@ impl ExecutionPlan for DistributedLeafExec { self.original.metrics() } - fn partition_statistics(&self, partition: Option) -> Result> { - self.original.partition_statistics(partition) + fn statistics_from_inputs( + &self, + _input_stats: &[Arc], + args: &StatisticsArgs, + ) -> Result> { + // `original` is not exposed through `children()`, so it needs its own walk. + StatisticsContext::new().compute(self.original.as_ref(), args) } } diff --git a/src/protocol/grpc/generated/worker.rs b/src/protocol/grpc/generated/worker.rs index d1f0fb4a9..4441b67fa 100644 --- a/src/protocol/grpc/generated/worker.rs +++ b/src/protocol/grpc/generated/worker.rs @@ -259,7 +259,7 @@ pub struct Metric { pub partition: ::core::option::Option, #[prost( oneof = "metric::Value", - tags = "10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34" + tags = "10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35" )] pub value: ::core::option::Option, } @@ -317,6 +317,8 @@ pub mod metric { CustomP99Latency(super::PercentileLatency), #[prost(message, tag = "34")] CustomMaxGauge(super::MaxGauge), + #[prost(message, tag = "35")] + PeakMemoryUsage(super::PeakMemoryUsage), } } /// A MetricsSet is a protobuf mirror of datafusion::physical_plan::metrics::MetricsSet. It represents @@ -352,6 +354,11 @@ pub struct SpilledRows { pub value: u64, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PeakMemoryUsage { + #[prost(uint64, tag = "1")] + pub value: u64, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CurrentMemoryUsage { #[prost(uint64, tag = "1")] pub value: u64, diff --git a/src/protocol/grpc/metrics_proto.rs b/src/protocol/grpc/metrics_proto.rs index 6db8cda92..f4eeda017 100644 --- a/src/protocol/grpc/metrics_proto.rs +++ b/src/protocol/grpc/metrics_proto.rs @@ -104,6 +104,11 @@ pub fn df_metric_to_proto(metric: Arc) -> Result Ok(pb::Metric { + value: Some(pb::metric::Value::PeakMemoryUsage(pb::PeakMemoryUsage { value: gauge.value() as u64 })), + partition, + labels, }), MetricValue::Count { name, count } => Ok(pb::Metric { value: Some(pb::metric::Value::Count(pb::NamedCount { @@ -346,6 +351,18 @@ pub fn metric_proto_to_df(metric: pb::Metric) -> Result, DataFusionE labels, ))) } + Some(pb::metric::Value::PeakMemoryUsage(memory)) => { + let gauge = Gauge::new(); + gauge.set(memory.value as usize); + Ok(Arc::new(Metric::new_with_labels( + MetricValue::PeakMemoryUsage { + name: Cow::Borrowed("peak_memory_usage"), + gauge, + }, + partition, + labels, + ))) + } Some(pb::metric::Value::Count(named_count)) => { let count = Count::new(); count.add(named_count.value as usize); @@ -646,6 +663,16 @@ mod tests { (MetricValue::CurrentMemoryUsage(orig), MetricValue::CurrentMemoryUsage(rt)) => { assert_eq!(orig.value(), rt.value()); } + ( + MetricValue::PeakMemoryUsage { + gauge: orig_gauge, .. + }, + MetricValue::PeakMemoryUsage { + gauge: rt_gauge, .. + }, + ) => { + assert_eq!(orig_gauge.value(), rt_gauge.value()); + } ( MetricValue::Count { name: n1, diff --git a/src/protocol/grpc/worker.proto b/src/protocol/grpc/worker.proto index 24f5c9871..82843f52d 100644 --- a/src/protocol/grpc/worker.proto +++ b/src/protocol/grpc/worker.proto @@ -253,6 +253,7 @@ message Metric { PercentileLatency custom_p95_latency = 32; PercentileLatency custom_p99_latency = 33; MaxGauge custom_max_gauge = 34; + PeakMemoryUsage peak_memory_usage = 35; } } @@ -282,6 +283,10 @@ message SpilledRows { uint64 value = 1; } +message PeakMemoryUsage { + uint64 value = 1; +} + message CurrentMemoryUsage { uint64 value = 1; } @@ -363,4 +368,4 @@ message PercentileLatency { message MaxGauge { string name = 1; uint64 value = 2; -} \ No newline at end of file +} diff --git a/src/stage.rs b/src/stage.rs index 7903a48a9..cd054c575 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -8,7 +8,8 @@ use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::metrics::{Label, Metric, MetricsSet}; use datafusion::physical_plan::{ - ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, displayable, + ColumnStatistics, ExecutionPlan, ExecutionPlanProperties, StatisticsArgs, StatisticsContext, + displayable, }; use itertools::Either; use std::collections::VecDeque; @@ -158,7 +159,10 @@ impl Stage { schema: SchemaRef, ) -> Result> { match self { - Stage::Local(local) => local.plan.partition_statistics(partition), + Stage::Local(local) => StatisticsContext::new().compute( + local.plan.as_ref(), + &StatisticsArgs::new().with_partition(partition), + ), Stage::Remote(remote) => { let Some(runtime_stats) = &remote.runtime_stats else { return Ok(Arc::new(Statistics::new_unknown(&schema))); diff --git a/src/test_utils/routing.rs b/src/test_utils/routing.rs index 114483040..7a1df81b6 100644 --- a/src/test_utils/routing.rs +++ b/src/test_utils/routing.rs @@ -14,7 +14,10 @@ use datafusion::{ }, prelude::Expr, }; -use datafusion_proto::{physical_plan::PhysicalExtensionCodec, protobuf::proto_error}; +use datafusion_proto::{ + physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}, + protobuf::proto_error, +}; use futures::stream; use prost::Message; use std::{fmt::Formatter, sync::Arc}; @@ -320,6 +323,7 @@ impl PhysicalExtensionCodec for URLEmitterExtensionCodec { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { if !inputs.is_empty() { return internal_err!( @@ -348,7 +352,12 @@ impl PhysicalExtensionCodec for URLEmitterExtensionCodec { )) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { let Some(exec) = node.downcast_ref::() else { return internal_err!("Expected URLEmitterExec, but was {}", node.name()); }; diff --git a/src/test_utils/test_work_unit_feed.rs b/src/test_utils/test_work_unit_feed.rs index e05f1cf82..7f4a5e1f0 100644 --- a/src/test_utils/test_work_unit_feed.rs +++ b/src/test_utils/test_work_unit_feed.rs @@ -21,7 +21,7 @@ use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{Count, ExecutionPlanMetricsSet, MetricBuilder}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; use datafusion_proto::protobuf::proto_error; use futures::StreamExt; use futures::stream::BoxStream; @@ -490,6 +490,7 @@ impl PhysicalExtensionCodec for TestWorkUnitFeedExecCodec { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { if !inputs.is_empty() { return internal_err!( @@ -518,7 +519,12 @@ impl PhysicalExtensionCodec for TestWorkUnitFeedExecCodec { ))) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { let Some(exec) = node.downcast_ref::() else { return internal_err!("Expected RowGeneratorExec, but was {}", node.name()); }; diff --git a/src/test_utils/work_unit_file_scan.rs b/src/test_utils/work_unit_file_scan.rs index 29245f1e2..6f6c36dd3 100644 --- a/src/test_utils/work_unit_file_scan.rs +++ b/src/test_utils/work_unit_file_scan.rs @@ -17,6 +17,7 @@ use datafusion::common::{Statistics, internal_err}; use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::{FileGroup, FileScanConfig, FileScanConfigBuilder}; use datafusion::datasource::source::DataSource; +use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::{EquivalenceProperties, LexOrdering}; use datafusion::physical_optimizer::PhysicalOptimizerRule; @@ -25,10 +26,11 @@ use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayFormatType, ExecutionPlan, Partitioning}; use datafusion_proto::physical_plan::{ - AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, + DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalPlanNodeExt, + PhysicalProtoConverterExtension, }; -use datafusion_proto::protobuf as df_proto; use datafusion_proto::protobuf::proto_error; +use datafusion_proto::{TryFromProto, protobuf as df_proto}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use prost::Message; @@ -76,8 +78,7 @@ impl WorkUnitFeedProvider for FileScanWorkUnitProvider { return Ok(futures::stream::empty().boxed()); }; let stream = futures::stream::iter(file_group.files().to_vec()).map(|file| { - let file_proto: df_proto::PartitionedFile = (&file) - .try_into() + let file_proto = df_proto::PartitionedFile::try_from_proto(&file) .map_err(|e| internal_datafusion_err!("{e}"))?; Ok(FileScanWorkUnit { file: Some(file_proto), @@ -131,8 +132,10 @@ impl DataSource for WorkUnitFileScanConfig { .map(move |work_unit| { let file = work_unit?.file.expect("missing file"); - let single_file_group = df_proto::FileGroup { files: vec![file] }; - let single_file_group = FileGroup::try_from(&single_file_group)?; + let df_file = + datafusion::datasource::listing::PartitionedFile::try_from_proto(&file) + .map_err(|e: DataFusionError| e)?; + let single_file_group = FileGroup::from(vec![df_file]); let new_config = FileScanConfigBuilder::from(inner.clone()) .with_file_groups(vec![single_file_group]) @@ -240,6 +243,7 @@ impl PhysicalExtensionCodec for WorkUnitFileScanCodec { buf: &[u8], inputs: &[Arc], ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { if !inputs.is_empty() { return internal_err!( @@ -252,8 +256,11 @@ impl PhysicalExtensionCodec for WorkUnitFileScanCodec { let plan_node = df_proto::PhysicalPlanNode::decode(&proto.inner[..]) .map_err(|e| proto_error(format!("Failed to decode template plan: {e}")))?; - let template_plan = - plan_node.try_into_physical_plan(ctx, &DefaultPhysicalExtensionCodec {})?; + let template_plan = plan_node.try_into_physical_plan_with_converter( + ctx, + &DefaultPhysicalExtensionCodec {}, + proto_converter, + )?; let Some(dse) = template_plan.downcast_ref::() else { return Err(proto_error( "Expected the WorkUnitFileScan template plan to be a DataSourceExec", @@ -274,7 +281,12 @@ impl PhysicalExtensionCodec for WorkUnitFileScanCodec { })) } - fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { let Some(dse) = node.downcast_ref::() else { return internal_err!( "Expected DataSourceExec wrapping a WorkUnitFileScanConfig, got {}", @@ -287,9 +299,10 @@ impl PhysicalExtensionCodec for WorkUnitFileScanCodec { // Encode the template DataSourceExec(FileScanConfig) as a regular // PhysicalPlanNode using DataFusion's default codec. - let plan_node = df_proto::PhysicalPlanNode::try_from_physical_plan( + let plan_node = df_proto::PhysicalPlanNode::try_from_physical_plan_with_converter( DataSourceExec::from_data_source(wfs.fsc.clone()), &DefaultPhysicalExtensionCodec {}, + proto_converter, )?; let mut inner_bytes = Vec::new(); plan_node.encode(&mut inner_bytes).map_err(|e| { diff --git a/src/worker/session_builder.rs b/src/worker/session_builder.rs index c0f0bf413..9aa84406e 100644 --- a/src/worker/session_builder.rs +++ b/src/worker/session_builder.rs @@ -25,18 +25,18 @@ pub trait WorkerSessionBuilder { /// # use datafusion::error::DataFusionError; /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; - /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; + /// # use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// #[derive(Debug)] /// struct CustomExecCodec; /// /// impl PhysicalExtensionCodec for CustomExecCodec { - /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext, ext: &dyn PhysicalProtoConverterExtension) -> datafusion::common::Result> { /// todo!() /// } /// - /// fn try_encode(&self, node: Arc, buf: &mut Vec) -> datafusion::common::Result<()> { + /// fn try_encode(&self, node: Arc, buf: &mut Vec, ext: &dyn PhysicalProtoConverterExtension) -> datafusion::common::Result<()> { /// todo!() /// } /// } diff --git a/tests/clickbench_plans_test.rs b/tests/clickbench_plans_test.rs index b9c32bb12..cd1f76d3e 100644 --- a/tests/clickbench_plans_test.rs +++ b/tests/clickbench_plans_test.rs @@ -157,14 +157,14 @@ mod tests { #[tokio::test] async fn test_clickbench_7() -> Result<()> { let display = test_clickbench_query("q7").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@1 DESC] │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*)] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*)] + │ SortExec: expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -185,16 +185,17 @@ mod tests { #[tokio::test] async fn test_clickbench_8() -> Result<()> { let display = test_clickbench_query("q8").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [u@1 DESC], fetch=10 │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[RegionID@0 as RegionID, count(alias1)@1 as u] - │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=6 │ RepartitionExec: partitioning=Hash([RegionID@0], 6), input_partitions=3 @@ -218,16 +219,17 @@ mod tests { #[tokio::test] async fn test_clickbench_9() -> Result<()> { let display = test_clickbench_query("q9").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)] - │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([RegionID@0], 9), input_partitions=3 @@ -248,8 +250,8 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [u@1 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u] + │ ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u] + │ SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── @@ -279,8 +281,8 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [u@2 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[u@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u] + │ ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u] + │ SortExec: TopK(fetch=10), expr=[count(alias1)@2 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── @@ -307,16 +309,17 @@ mod tests { #[tokio::test] async fn test_clickbench_12() -> Result<()> { let display = test_clickbench_query("q12").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@1 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 6), input_partitions=3 @@ -338,8 +341,8 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [u@1 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u] + │ SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── @@ -366,16 +369,17 @@ mod tests { #[tokio::test] async fn test_clickbench_14() -> Result<()> { let display = test_clickbench_query("q14").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 6), input_partitions=3 @@ -394,16 +398,17 @@ mod tests { #[tokio::test] async fn test_clickbench_15() -> Result<()> { let display = test_clickbench_query("q15").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@1 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*)] - │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([UserID@0], 9), input_partitions=3 @@ -421,16 +426,17 @@ mod tests { #[tokio::test] async fn test_clickbench_16() -> Result<()> { let display = test_clickbench_query("q16").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [count(*)@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[count(*)@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*)] - │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 9), input_partitions=3 @@ -481,10 +487,11 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[count(*)@3 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*)] - │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@3 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 9), input_partitions=3 @@ -570,10 +577,10 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[EventTime@4 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: CAST(URL@13 AS Utf8View) LIKE %google% │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -614,10 +621,10 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: SearchPhrase@0 != │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] └────────────────────────────────────────────────── "); Ok(()) @@ -626,20 +633,22 @@ mod tests { #[tokio::test] async fn test_clickbench_26() -> Result<()> { let display = test_clickbench_query("q26").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase] │ SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=10 │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=12 - │ SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] - │ FilterExec: SearchPhrase@1 != , projection=[SearchPhrase@1, EventTime@0] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ LocalLimitExec: fetch=10 + │ ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime] + │ SortExec: TopK(fetch=10), expr=[EventTime@0 ASC NULLS LAST, SearchPhrase@1 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: SearchPhrase@1 != + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:.., /testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] └────────────────────────────────────────────────── "); Ok(()) @@ -648,17 +657,18 @@ mod tests { #[tokio::test] async fn test_clickbench_27() -> Result<()> { let display = test_clickbench_query("q27").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [l@1 DESC], fetch=25 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=25 │ ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c] - │ FilterExec: count(Int64(1))@2 > 100000 - │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=25), expr=[avg(length(hits.URL))@1 DESC], preserve_partitioning=[true] + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([CounterID@0], 6), input_partitions=3 @@ -683,11 +693,12 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=25 │ ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)] - │ FilterExec: count(Int64(1))@2 > 100000 - │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=25), expr=[avg(length(hits.Referer))@1 DESC], preserve_partitioning=[true] + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 6), input_partitions=3 @@ -728,16 +739,17 @@ mod tests { #[tokio::test] async fn test_clickbench_30() -> Result<()> { let display = test_clickbench_query("q30").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 6), input_partitions=3 @@ -756,16 +768,17 @@ mod tests { #[tokio::test] async fn test_clickbench_31() -> Result<()> { let display = test_clickbench_query("q31").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] - │ AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([WatchID@0, ClientIP@1], 6), input_partitions=3 @@ -784,16 +797,17 @@ mod tests { #[tokio::test] async fn test_clickbench_32() -> Result<()> { let display = test_clickbench_query("q32").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@2 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] - │ AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([WatchID@0, ClientIP@1], 9), input_partitions=3 @@ -811,16 +825,17 @@ mod tests { #[tokio::test] async fn test_clickbench_33() -> Result<()> { let display = test_clickbench_query("q33").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c@1 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=3 - │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c] - │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([URL@0], 9), input_partitions=3 @@ -893,16 +908,17 @@ mod tests { #[tokio::test] async fn test_clickbench_36() -> Result<()> { let display = test_clickbench_query("q36").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 @@ -921,16 +937,17 @@ mod tests { #[tokio::test] async fn test_clickbench_37() -> Result<()> { let display = test_clickbench_query("q37").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10 │ ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([Title@0], 6), input_partitions=3 @@ -949,17 +966,18 @@ mod tests { #[tokio::test] async fn test_clickbench_38() -> Result<()> { let display = test_clickbench_query("q38").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ GlobalLimitExec: skip=1000, fetch=10 │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=1010 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=1010 │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=1010), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 @@ -985,10 +1003,11 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=1010 │ ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=1010), expr=[count(Int64(1))@5 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 6), input_partitions=3 @@ -1007,17 +1026,18 @@ mod tests { #[tokio::test] async fn test_clickbench_40() -> Result<()> { let display = test_clickbench_query("q40").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ GlobalLimitExec: skip=100, fetch=10 │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=110 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=110 │ ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=110), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 6), input_partitions=3 @@ -1036,17 +1056,18 @@ mod tests { #[tokio::test] async fn test_clickbench_41() -> Result<()> { let display = test_clickbench_query("q41").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ GlobalLimitExec: skip=10000, fetch=10 │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=10010 │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=10010 │ ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=10010), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 6), input_partitions=3 diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index 0f4502fa9..50ccb0fe9 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -14,7 +14,9 @@ mod tests { use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{DistributedExt, WorkerQueryContext}; - use datafusion_proto::physical_plan::PhysicalExtensionCodec; + use datafusion_proto::physical_plan::{ + PhysicalExtensionCodec, PhysicalProtoConverterExtension, + }; use futures::TryStreamExt; use prost::Message; use std::fmt::Formatter; @@ -201,6 +203,7 @@ mod tests { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result> { let _node = CustomConfigExtensionRequiredExecProto::decode(buf) .map_err(|err| internal_datafusion_err!("{err}"))?; @@ -221,6 +224,7 @@ mod tests { &self, _node: Arc, buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result<()> { CustomConfigExtensionRequiredExecProto::default() .encode(buf) diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index a3eb3d9ff..7c23c8d8b 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -13,7 +13,9 @@ mod tests { use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{DistributedExt, WorkerQueryContext, assert_snapshot}; - use datafusion_proto::physical_plan::PhysicalExtensionCodec; + use datafusion_proto::physical_plan::{ + PhysicalExtensionCodec, PhysicalProtoConverterExtension, + }; use datafusion_proto::protobuf::proto_error; use futures::TryStreamExt; use prost::Message; @@ -139,6 +141,7 @@ mod tests { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result> { let _node = CustomPassThroughExecProto::decode(buf) .map_err(|err| proto_error(format!("{err}")))?; @@ -157,6 +160,7 @@ mod tests { &self, node: Arc, buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result<()> { let Some(_plan) = node.downcast_ref::() else { return Err(proto_error(format!( diff --git a/tests/distributed_aggregation.rs b/tests/distributed_aggregation.rs index 7ba4d512b..ccb514c84 100644 --- a/tests/distributed_aggregation.rs +++ b/tests/distributed_aggregation.rs @@ -35,10 +35,10 @@ mod tests { let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); assert_snapshot!(physical_str, - @r" + @" SortPreservingMergeExec: [count(*)@0 ASC NULLS LAST] - SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=3 AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] @@ -53,8 +53,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── diff --git a/tests/distributed_unions.rs b/tests/distributed_unions.rs index e80588d8b..bc767543b 100644 --- a/tests/distributed_unions.rs +++ b/tests/distributed_unions.rs @@ -49,8 +49,8 @@ mod tests { │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[MaxTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: MaxTemp@0 < 30 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] @@ -100,13 +100,13 @@ mod tests { │ FilterExec: MinTemp@0 > 20 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 20, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 20, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[MaxTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: MaxTemp@0 < 25 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 25, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 25, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[Temp9am@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: Temp9am@0 > 15 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] @@ -160,23 +160,23 @@ mod tests { │ FilterExec: MinTemp@0 > 10 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[MaxTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: MaxTemp@0 < 30 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[Temp9am@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: Temp9am@0 > 15 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[Temp3pm@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: Temp3pm@0 < 25 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: expr=[Rainfall@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] │ FilterExec: Rainfall@0 > 5 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] @@ -226,7 +226,7 @@ mod tests { let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); assert_snapshot!(physical_distributed_str, - @r" + @" ┌───── DistributedExec │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] │ [Stage 7] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 @@ -236,9 +236,9 @@ mod tests { │ SortExec: TopK(fetch=1000000), expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[false] │ CoalescePartitionsExec │ [Stage 3] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 - │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[false] - │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] - │ CoalescePartitionsExec: fetch=1000000 + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ SortExec: TopK(fetch=1000000), expr=[Temp9am@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[false] + │ CoalescePartitionsExec │ [Stage 6] => NetworkCoalesceExec: output_partitions=2, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=2, partitions=2 @@ -252,39 +252,38 @@ mod tests { ┌───── Stage 1 ── tasks=3, partitions=9 │ FilterExec: MinTemp@0 > 10, fetch=1000000 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=9 │ FilterExec: MaxTemp@0 < 30, fetch=1000000 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=2, partitions=2 - │ LocalLimitExec: fetch=1000000 - │ DistributedUnionExec: t0:[c0] t1:[c1] + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ CoalescePartitionsExec: fetch=1000000 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[Temp3pm@0 as Temp9am, RainToday@1 as RainToday] │ CoalescePartitionsExec: fetch=1000000 - │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 - │ ProjectionExec: expr=[Temp3pm@0 as Temp9am, RainToday@1 as RainToday] - │ CoalescePartitionsExec: fetch=1000000 - │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=3, partitions=9 │ FilterExec: Temp9am@0 > 15, fetch=1000000 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=3, partitions=9 │ FilterExec: Temp3pm@0 < 25, fetch=1000000 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] └────────────────────────────────────────────────── ", ); diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index 44908e712..052895940 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -13,7 +13,9 @@ mod tests { use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{DistributedExt, WorkerQueryContext}; - use datafusion_proto::physical_plan::PhysicalExtensionCodec; + use datafusion_proto::physical_plan::{ + PhysicalExtensionCodec, PhysicalProtoConverterExtension, + }; use datafusion_proto::protobuf::proto_error; use futures::{TryStreamExt, stream}; use prost::Message; @@ -146,6 +148,7 @@ mod tests { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result> { let node = ErrorThrowingExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; @@ -167,6 +170,7 @@ mod tests { &self, node: Arc, buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> datafusion::common::Result<()> { let Some(plan) = node.downcast_ref::() else { return Err(proto_error(format!( diff --git a/tests/join.rs b/tests/join.rs index 1f469224e..b28cc037d 100644 --- a/tests/join.rs +++ b/tests/join.rs @@ -116,11 +116,11 @@ mod tests { │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@3, f_dkey@2)], projection=[f_dkey@6, timestamp@4, value@5, env@0, service@1, host@2] │ FilterExec: service@1 = log │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [], []]}, projection=[env, service, host, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet], [], []]}, projection=[env, service, host, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, host, d_dkey], output_partitioning=Hash([d_dkey@3], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=D/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, host, d_dkey], output_partitioning=Hash([d_dkey@3], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet └────────────────────────────────────────────────── "); @@ -197,11 +197,11 @@ mod tests { │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] │ FilterExec: service@1 = log, projection=[env@0, d_dkey@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [], []]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet], [], []]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=D/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet └────────────────────────────────────────────────── "#); @@ -266,8 +266,8 @@ mod tests { assert_snapshot!(&distributed_plan, @r#" ┌───── DistributedExec │ SortPreservingMergeExec: [env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST] - │ SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value] + │ ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value] + │ SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, time_bin@1 as time_bin], aggr=[avg(a.max_bin_value)] │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 └────────────────────────────────────────────────── @@ -279,11 +279,11 @@ mod tests { │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3] │ FilterExec: service@1 = log, projection=[env@0, d_dkey@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [], []]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet], [], []]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=B/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=C/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet:.., /testdata/join/parquet/dim/d_dkey=D/data0.parquet:..], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet:..]]}, projection=[env, service, d_dkey], output_partitioning=Hash([d_dkey@2], 4), file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet - │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet], [], []]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet + │ t0: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet + │ t1: DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=B/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=A/data0.parquet:..], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet:..]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], output_partitioning=Hash([f_dkey@2], 4), file_type=parquet └────────────────────────────────────────────────── "#); diff --git a/tests/multi_task_collect_join_repros.rs b/tests/multi_task_collect_join_repros.rs index 46e01791f..23580e746 100644 --- a/tests/multi_task_collect_join_repros.rs +++ b/tests/multi_task_collect_join_repros.rs @@ -80,10 +80,10 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([id@0], 12), input_partitions=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ") } @@ -100,10 +100,10 @@ mod tests { .await .unwrap(); assert_snapshot!(display_plan_ascii(plan.as_ref(), false), @" - HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)] + HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware CoalescePartitionsExec DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet, /target/multi_task_collect_join_repros/build_side/part-1.parquet], [/target/multi_task_collect_join_repros/build_side/part-2.parquet, /target/multi_task_collect_join_repros/build_side/part-3.parquet]]}, projection=[id], file_type=parquet - DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet, /target/multi_task_collect_join_repros/probe_side/part-1.parquet], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet, /target/multi_task_collect_join_repros/probe_side/part-3.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 >= 0 AND id@0 <= 99 AND id@0 IN (SET) ([]) ], pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 >= 0 AND id_null_count@1 != row_count@2 AND id_min@3 <= 99, required_guarantees=[id in (0, 1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 3, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 4, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 5, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 6, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 7, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 8, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 9, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99)] + DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet, /target/multi_task_collect_join_repros/probe_side/part-1.parquet], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet, /target/multi_task_collect_join_repros/probe_side/part-3.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 IS NULL OR id@0 >= 0 AND id@0 <= 99 AND id@0 IN (SET) ([]) ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@0 > 0 OR id_null_count@0 != row_count@2 AND id_max@1 >= 0 AND id_null_count@0 != row_count@2 AND id_min@3 <= 99, required_guarantees=[] ") } @@ -111,7 +111,6 @@ mod tests { /// by a correlated EXISTS with a non-equi predicate (`p.id > b.id - 1 AND p.id < b.id + 1` /// is `p.id = b.id` for integers, but expressed as inequalities so no hash join is possible). #[tokio::test] - #[ignore = "Until we upgrade to DF 54.1.0 or greater, this is flaky. See apache/datafusion#22791"] async fn nested_loop_left_semi_join_is_correct() { let plan = assert_distributed_matches_single_node( "SELECT b.id FROM build_side b WHERE EXISTS ( \ @@ -148,7 +147,6 @@ mod tests { /// broadcast orientation can ever be correct. Single-node: every row matches, no NULL /// padding. Distributed: cross-task matches are lost and spurious NULL-padded rows appear. #[tokio::test] - #[ignore = "Until we upgrade to DF 54.1.0 or greater, this is flaky. See apache/datafusion#22791"] async fn nested_loop_full_join_is_correct() { let plan = assert_distributed_matches_single_node( "SELECT b.id, p.id FROM build_side b FULL JOIN probe_side p \ @@ -251,8 +249,8 @@ mod tests { .unwrap(); assert_snapshot!(display_plan_ascii(plan.as_ref(), false), @" ┌───── DistributedExec - │ SortExec: expr=[bid@0 ASC NULLS LAST, pid@1 ASC NULLS LAST], preserve_partitioning=[false] - │ ProjectionExec: expr=[id@0 as bid, id@1 as pid] + │ ProjectionExec: expr=[id@0 as bid, id@1 as pid] + │ SortExec: expr=[id@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false] │ HashJoinExec: mode=CollectLeft, join_type=Full, on=[(id@0, id@0)] │ SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=5 │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 @@ -262,18 +260,18 @@ mod tests { ┌───── Stage 1 ── tasks=4, partitions=8 │ SortExec: TopK(fetch=5), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet:.., /target/multi_task_collect_join_repros/build_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-2.parquet:.., /target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-1.parquet:.., /target/multi_task_collect_join_repros/build_side/part-2.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] + │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet:.., /target/multi_task_collect_join_repros/build_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-2.parquet:.., /target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-1.parquet:.., /target/multi_task_collect_join_repros/build_side/part-2.parquet:..], [/target/multi_task_collect_join_repros/build_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=8 │ SortExec: TopK(fetch=1000000), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] + │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ") } @@ -296,8 +294,8 @@ mod tests { .unwrap(); assert_snapshot!(display_plan_ascii(plan.as_ref(), false), @" ┌───── DistributedExec - │ SortExec: expr=[bid@0 ASC NULLS LAST, pid@1 ASC NULLS LAST], preserve_partitioning=[false] - │ ProjectionExec: expr=[id@0 as bid, id@1 as pid] + │ ProjectionExec: expr=[id@0 as bid, id@1 as pid] + │ SortExec: expr=[id@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false] │ NestedLoopJoinExec: join_type=Left, filter=id@1 <= id@0 │ CoalescePartitionsExec │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -316,10 +314,10 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=8 │ SortExec: TopK(fetch=1000000), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] - │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST] + │ t0: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-3.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-1.parquet:..], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet:..]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ") } diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index b83329bdc..926053815 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -185,6 +185,7 @@ mod tests { buf: &[u8], inputs: &[Arc], _ctx: &TaskContext, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, ) -> datafusion::common::Result> { let _node = PassThroughExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; @@ -203,6 +204,7 @@ mod tests { &self, node: Arc, buf: &mut Vec, + _ext: &dyn datafusion_proto::physical_plan::PhysicalProtoConverterExtension, ) -> datafusion::common::Result<()> { let Some(_plan) = node.downcast_ref::() else { return Err(proto_error(format!( diff --git a/tests/task_estimator_test.rs b/tests/task_estimator_test.rs index 9d873a8f4..c0eaa96b9 100644 --- a/tests/task_estimator_test.rs +++ b/tests/task_estimator_test.rs @@ -267,8 +267,8 @@ mod tests { │ [Stage 3] => NetworkCoalesceExec: output_partitions=15, input_tasks=5 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=5, partitions=3 - │ SortExec: expr=[left_index@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[task_count@0 as task_count, task_index@1 as left_index, tag@2 as left_tag, worker_url@3 as worker_left, task_index@4 as right_index, tag@5 as right_tag, worker_url@6 as worker_right] + │ ProjectionExec: expr=[task_count@0 as task_count, task_index@1 as left_index, tag@2 as left_tag, worker_url@3 as worker_left, task_index@4 as right_index, tag@5 as right_tag, worker_url@6 as worker_right] + │ SortExec: expr=[task_index@1 ASC NULLS LAST], preserve_partitioning=[true] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(task_index@1, task_index@0)] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=5 │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=5 @@ -332,8 +332,8 @@ mod tests { │ [Stage 3] => NetworkCoalesceExec: output_partitions=15, input_tasks=5 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=5, partitions=3 - │ SortExec: expr=[left_index@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[task_count@0 as task_count, task_index@1 as left_index, tag@2 as left_tag, worker_url@3 as worker_left, task_index@4 as right_index, tag@5 as right_tag, worker_url@6 as worker_right] + │ ProjectionExec: expr=[task_count@0 as task_count, task_index@1 as left_index, tag@2 as left_tag, worker_url@3 as worker_left, task_index@4 as right_index, tag@5 as right_tag, worker_url@6 as worker_right] + │ SortExec: expr=[task_index@1 ASC NULLS LAST], preserve_partitioning=[true] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(task_index@1, task_index@0)] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=5 │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=5 diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs index bc00167e5..12bf66524 100644 --- a/tests/tpcds_plans_test.rs +++ b/tests/tpcds_plans_test.rs @@ -21,31 +21,27 @@ mod tests { #[tokio::test] async fn test_tpcds_1() -> Result<()> { let display = test_tpcds_query("q1").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST], fetch=100 - │ [Stage 9] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=3 + ┌───── Stage 10 ── tasks=4, partitions=3 │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_store_sk@0, ctr_store_sk@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@2] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_store_sk@0 as ctr_store_sk] - │ AggregateExec: mode=FinalPartitioned, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ctr_store_sk@0, ctr_store_sk@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@2] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=1 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 5 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ctr_store_sk@0], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, c_customer_sk@0)], projection=[ctr_store_sk@1, ctr_total_return@2, c_customer_id@4] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -73,50 +69,54 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=2, partitions=12 - │ RepartitionExec: partitioning=Hash([ctr_store_sk@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] - │ ProjectionExec: expr=[sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] - │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sr_returned_date_sk@0)], projection=[sr_customer_sk@2, sr_store_sk@3, sr_return_amt@4] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 9 ── tasks=1, partitions=12 + │ RepartitionExec: partitioning=Hash([ctr_store_sk@1], 12), input_partitions=3 + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_store_sk@0 as ctr_store_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── tasks=2, partitions=3 + │ RepartitionExec: partitioning=Hash([ctr_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 7 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sr_returned_date_sk@0)], projection=[sr_customer_sk@2, sr_store_sk@3, sr_return_amt@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── "); Ok(()) } @@ -143,40 +143,39 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_year@1 = 2001, projection=[d_week_seq@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_week_seq@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sold_date_sk@0)], projection=[sales_price@4, d_week_seq@1, d_day_name@2] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=6 │ RepartitionExec: partitioning=Hash([z.d_week_seq2 - Int64(53)@8], 6), input_partitions=3 @@ -185,40 +184,39 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_year@1 = 2002, projection=[d_week_seq@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_week_seq@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN wscs.sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sold_date_sk@0)], projection=[sales_price@4, d_week_seq@1, d_day_name@2] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "#); Ok(()) @@ -226,16 +224,17 @@ mod tests { #[tokio::test] async fn test_tpcds_3() -> Result<()> { let display = test_tpcds_query("q3").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as sum_agg] - │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[d_year@0 ASC NULLS LAST, sum(store_sales.ss_ext_sales_price)@3 DESC, i_brand_id@2 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 6), input_partitions=3 @@ -247,30 +246,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manufact_id@3 = 128, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 11, projection=[d_date_sk@0, d_year@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] └────────────────────────────────────────────────── "); Ok(()) @@ -278,18 +275,18 @@ mod tests { #[tokio::test] async fn test_tpcds_4() -> Result<()> { let display = test_tpcds_query("q4").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 24 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END > CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END, projection=[customer_id@1, customer_first_name@2, customer_last_name@3, customer_preferred_cust_flag@4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@0 > 0.000000 THEN year_total@1 / year_total@0 END > CASE WHEN year_total@2 > 0.000000 THEN year_total@3 / year_total@2 END, projection=[customer_id@1, customer_first_name@2, customer_last_name@3, customer_preferred_cust_flag@4] │ CoalescePartitionsExec │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as year_total] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Some(2),20,0) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / 2) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 20 ── tasks=2, partitions=12 @@ -297,53 +294,52 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[customer_id@2, customer_id@3, customer_first_name@4, customer_last_name@5, customer_preferred_cust_flag@6, year_total@7, year_total@8, year_total@1] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END, projection=[customer_id@0, customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5, year_total@7, year_total@9] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > 0.000000 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > 0.000000 THEN year_total@1 / year_total@0 END, projection=[customer_id@0, customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5, year_total@7, year_total@9] │ CoalescePartitionsExec │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as year_total] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Some(2),20,0) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / 2) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 as year_total] - │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 > 0.000000 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Some(2),20,0) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / 2) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Some(2),20,0) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / 2) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, ws_ext_discount_amt@10, ws_ext_sales_price@11, ws_ext_wholesale_cost@12, ws_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@8, ws_ext_discount_amt@10, ws_ext_sales_price@11, ws_ext_wholesale_cost@12, ws_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 @@ -354,189 +350,184 @@ mod tests { │ CoalescePartitionsExec │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as year_total] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Some(2),20,0) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / 2) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 as year_total] - │ FilterExec: sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ FilterExec: sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 > 0.000000 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Some(2),20,0) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / 2) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Some(2),20,0) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / 2) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, cs_ext_discount_amt@10, cs_ext_sales_price@11, cs_ext_wholesale_cost@12, cs_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@8, cs_ext_discount_amt@10, cs_ext_sales_price@11, cs_ext_wholesale_cost@12, cs_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 as year_total] - │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 > 0.000000 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Some(2),20,0) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / 2) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Some(2),20,0) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / 2) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, ss_ext_discount_amt@10, ss_ext_sales_price@11, ss_ext_wholesale_cost@12, ss_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@8, ss_ext_discount_amt@10, ss_ext_sales_price@11, ss_ext_wholesale_cost@12, ss_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Some(2),20,0) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / 2) as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, ss_ext_discount_amt@10, ss_ext_sales_price@11, ss_ext_wholesale_cost@12, ss_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@8, ss_ext_discount_amt@10, ss_ext_sales_price@11, ss_ext_wholesale_cost@12, ss_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 14 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Some(2),20,0) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / 2) as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, cs_ext_discount_amt@10, cs_ext_sales_price@11, cs_ext_wholesale_cost@12, cs_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@8, cs_ext_discount_amt@10, cs_ext_sales_price@11, cs_ext_wholesale_cost@12, cs_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 18 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 18 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 23 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Some(2),20,0) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / 2) as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@7)], projection=[c_customer_id@2, c_first_name@3, c_last_name@4, c_preferred_cust_flag@5, c_birth_country@6, c_login@7, c_email_address@8, ws_ext_discount_amt@10, ws_ext_sales_price@11, ws_ext_wholesale_cost@12, ws_ext_list_price@13, d_year@1] │ CoalescePartitionsExec │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@8, ws_ext_discount_amt@10, ws_ext_sales_price@11, ws_ext_wholesale_cost@12, ws_ext_list_price@13] │ CoalescePartitionsExec - │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 21 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 22 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 22 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -544,7 +535,7 @@ mod tests { #[tokio::test] async fn test_tpcds_5() -> Result<()> { let display = test_tpcds_query("q5").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] @@ -577,11 +568,11 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, ss_net_profit@22 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, ss_net_profit@22 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, ss_net_profit@22 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, ss_net_profit@22 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@11 as return_amt, sr_net_loss@19 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@11 as return_amt, sr_net_loss@19 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, sr_return_amt@11 as return_amt, sr_net_loss@19 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, sr_return_amt@11 as return_amt, sr_net_loss@19 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -594,12 +585,11 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 @@ -607,12 +597,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(page_sk@0, cp_catalog_page_sk@0)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -621,21 +610,20 @@ mod tests { │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_catalog_page_sk@12 as page_sk, cr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_catalog_page_sk@12 as page_sk, cr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_catalog_page_sk@12 as page_sk, cr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_catalog_page_sk@12 as page_sk, cr_returned_date_sk@0 as date_sk, 0.00 as sales_price, 0.00 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=3 @@ -648,9 +636,9 @@ mod tests { │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[ws_web_site_sk@0 as wsr_web_site_sk, wr_returned_date_sk@1 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, wr_return_amt@2 as return_amt, wr_net_loss@3 as net_loss] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, 0.00 as return_amt, 0.00 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ ProjectionExec: expr=[ws_web_site_sk@0 as wsr_web_site_sk, wr_returned_date_sk@1 as date_sk, 0.00 as sales_price, 0.00 as profit, wr_return_amt@2 as return_amt, wr_net_loss@3 as net_loss] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)], projection=[ws_web_site_sk@6, wr_returned_date_sk@0, wr_return_amt@3, wr_net_loss@4] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -666,28 +654,27 @@ mod tests { ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 6), input_partitions=2 + │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 6), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 6), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -695,124 +682,125 @@ mod tests { #[tokio::test] async fn test_tpcds_6() -> Result<()> { let display = test_tpcds_query("q6").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [cnt@1 ASC, state@0 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[cnt@1 ASC, state@0 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[ca_state@0 as state, count(Int64(1))@1 as cnt] - │ FilterExec: count(Int64(1))@1 >= 10 - │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=3 - │ RepartitionExec: partitioning=Hash([ca_state@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] - │ FilterExec: CAST(i_current_price@1 AS Decimal128(30, 15)) > CAST(1.2 * avg(j.i_current_price)@2 AS Decimal128(30, 15)), projection=[ca_state@0] - │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(i_category@1, i_category@2)], projection=[ca_state@2, i_current_price@3, avg(j.i_current_price)@0] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_month_seq@0, d_month_seq@1)], projection=[ca_state@1, i_current_price@3, i_category@4] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=1 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ca_state@3, d_month_seq@5, i_current_price@1, i_category@2] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@1)], projection=[ca_state@2, ss_item_sk@4, d_month_seq@1] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@1, ss_customer_sk@2)], projection=[ca_state@0, ss_sold_date_sk@2, ss_item_sk@3] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 10 ── tasks=2, partitions=3 + │ LocalLimitExec: fetch=100 + │ ProjectionExec: expr=[ca_state@0 as state, count(Int64(1))@1 as cnt] + │ SortExec: TopK(fetch=100), expr=[count(Int64(1))@1 ASC, ca_state@0 ASC], preserve_partitioning=[true] + │ FilterExec: count(Int64(1))@1 >= 10 + │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=2, partitions=24 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ ProjectionExec: expr=[CAST(avg(j.i_current_price)@1 AS Float64) as avg(j.i_current_price), i_category@0 as i_category] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[avg(j.i_current_price)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + ┌───── Stage 9 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([ca_state@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_category@1, i_category@2)], filter=CAST(i_current_price@0 AS Decimal128(30, 15)) > CAST(1.2 * avg(j.i_current_price)@1 AS Decimal128(30, 15)), projection=[ca_state@0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_month_seq@0, d_month_seq@1)], projection=[ca_state@1, i_current_price@3, i_category@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=1 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ca_state@3, d_month_seq@5, i_current_price@1, i_category@2] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@1)], projection=[ca_state@2, ss_item_sk@4, d_month_seq@1] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@1, ss_customer_sk@2)], projection=[ca_state@0, ss_sold_date_sk@2, ss_item_sk@3] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([i_category@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category], aggr=[avg(j.i_current_price)] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet + ┌───── Stage 2 ── tasks=2, partitions=24 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[CAST(avg(j.i_current_price)@1 AS Float64) as avg(j.i_current_price), i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[avg(j.i_current_price)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=1, partitions=12 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ AggregateExec: mode=FinalPartitioned, gby=[d_month_seq@0 as d_month_seq], aggr=[] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=3 - │ RepartitionExec: partitioning=Hash([d_month_seq@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_month_seq@0 as d_month_seq], aggr=[] - │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 1, projection=[d_month_seq@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + ┌───── Stage 1 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([i_category@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category], aggr=[avg(j.i_current_price)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_current_price, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=1, partitions=12 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ AggregateExec: mode=FinalPartitioned, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@1)], projection=[ca_state@1, c_customer_sk@2] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=3 + │ RepartitionExec: partitioning=Hash([d_month_seq@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 1, projection=[d_month_seq@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@1)], projection=[ca_state@1, c_customer_sk@2] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_7() -> Result<()> { let display = test_tpcds_query("q7").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(store_sales.ss_quantity)@1 as agg1, avg(store_sales.ss_list_price)@2 as agg2, avg(store_sales.ss_coupon_amt)@3 as agg3, avg(store_sales.ss_sales_price)@4 as agg4] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -822,7 +810,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_promo_sk@3, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_promo_sk@3, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -840,23 +828,22 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:..]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet:..]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -865,15 +852,15 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -905,10 +892,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=1, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -924,12 +911,11 @@ mod tests { │ AggregateExec: mode=Partial, gby=[ca_zip@0 as ca_zip], aggr=[] │ ProjectionExec: expr=[substr(ca_zip@0, 1, 5) as ca_zip] │ FilterExec: substr(ca_zip@0, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 @@ -944,40 +930,37 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@0, ca_address_sk@0)], projection=[ca_zip@2] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: c_preferred_cust_flag@1 = Y, projection=[c_current_addr_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 1998, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] └────────────────────────────────────────────────── "); Ok(()) @@ -1211,126 +1194,118 @@ mod tests { assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], fetch=100 - │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@8 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@8 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@8 as cnt3, cd_dep_count@5 as cd_dep_count, count(Int64(1))@8 as cnt4, cd_dep_employed_count@6 as cd_dep_employed_count, count(Int64(1))@8 as cnt5, cd_dep_college_count@7 as cd_dep_college_count, count(Int64(1))@8 as cnt6] + │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@8 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@8 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@8 as cnt3, cd_dep_count@5 as cd_dep_count, count(Int64(1))@8 as cnt4, cd_dep_employed_count@6 as cd_dep_employed_count, count(Int64(1))@8 as cnt5, cd_dep_college_count@7 as cd_dep_college_count, count(Int64(1))@8 as cnt6] + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@3 ASC NULLS LAST, cd_credit_rating@4 ASC NULLS LAST, cd_dep_count@5 ASC NULLS LAST, cd_dep_employed_count@6 ASC NULLS LAST, cd_dep_college_count@7 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=3 + ┌───── Stage 11 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] │ FilterExec: mark@8 OR mark@9, projection=[cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7] │ HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(c_customer_sk@0, cs_ship_customer_sk@0)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8, mark@9, mark@10] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=12 + ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(c_customer_sk@0, ws_bill_customer_sk@0)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, c_customer_sk@0)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@1] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7, cd_dep_count@8, cd_dep_employed_count@9, cd_dep_college_count@10] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7, cd_dep_count@8, cd_dep_employed_count@9, cd_dep_college_count@10] + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 + ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ FilterExec: ca_county@1 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_county@1 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@2] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@2] + ┌───── Stage 7 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 + ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=12 + ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_ship_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=48 + ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── "); Ok(()) @@ -1338,14 +1313,14 @@ mod tests { #[tokio::test] async fn test_tpcds_11() -> Result<()> { let display = test_tpcds_query("q11").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),18,2 THEN CAST(year_total@3 AS Float64) / CAST(year_total@2 AS Float64) ELSE 0 END > CASE WHEN year_total@0 > Some(0),18,2 THEN CAST(year_total@1 AS Float64) / CAST(year_total@0 AS Float64) ELSE 0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > 0.00 THEN CAST(year_total@3 AS Float64) / CAST(year_total@2 AS Float64) ELSE 0 END > CASE WHEN year_total@0 > 0.00 THEN CAST(year_total@1 AS Float64) / CAST(year_total@0 AS Float64) ELSE 0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5] │ CoalescePartitionsExec │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as year_total] @@ -1367,7 +1342,7 @@ mod tests { ┌───── Stage 4 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 as year_total] - │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 > Some(0),18,2 + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 > 0.00 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -1380,35 +1355,34 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@8, ws_ext_discount_amt@10, ws_ext_list_price@11] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 as year_total] - │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 > Some(0),18,2 + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 > 0.00 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -1421,30 +1395,29 @@ mod tests { │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@8, ss_ext_discount_amt@10, ss_ext_list_price@11] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 @@ -1454,30 +1427,29 @@ mod tests { │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@8, ss_ext_discount_amt@10, ss_ext_list_price@11] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=3 @@ -1487,30 +1459,29 @@ mod tests { │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@8, ws_ext_discount_amt@10, ws_ext_list_price@11] │ CoalescePartitionsExec - │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 14 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -1542,30 +1513,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] └────────────────────────────────────────────────── "#); Ok(()) @@ -1585,35 +1554,33 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, ss_ext_wholesale_cost@4] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], filter=(ca_state@1 = TX OR ca_state@1 = OH) AND ss_net_profit@0 >= Some(10000),7,2 AND ss_net_profit@0 <= Some(20000),7,2 OR (ca_state@1 = OR OR ca_state@1 = NM OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),7,2 AND ss_net_profit@0 <= Some(30000),7,2 OR (ca_state@1 = VA OR ca_state@1 = TX OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),7,2 AND ss_net_profit@0 <= Some(25000),7,2, projection=[ss_sold_date_sk@2, ss_quantity@4, ss_ext_sales_price@5, ss_ext_wholesale_cost@6] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], filter=(ca_state@1 = TX OR ca_state@1 = OH) AND ss_net_profit@0 >= 100.00 AND ss_net_profit@0 <= 200.00 OR (ca_state@1 = OR OR ca_state@1 = NM OR ca_state@1 = KY) AND ss_net_profit@0 >= 150.00 AND ss_net_profit@0 <= 300.00 OR (ca_state@1 = VA OR ca_state@1 = TX OR ca_state@1 = MS) AND ss_net_profit@0 >= 50.00 AND ss_net_profit@0 <= 250.00, projection=[ss_sold_date_sk@2, ss_quantity@4, ss_ext_sales_price@5, ss_ext_wholesale_cost@6] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, ss_hdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),7,2 AND ss_sales_price@0 <= Some(15000),7,2 AND hd_dep_count@3 = 3 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),7,2 AND ss_sales_price@0 <= Some(10000),7,2 AND hd_dep_count@3 = 1 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),7,2 AND ss_sales_price@0 <= Some(20000),7,2 AND hd_dep_count@3 = 1, projection=[ss_sold_date_sk@2, ss_addr_sk@4, ss_quantity@5, ss_ext_sales_price@7, ss_ext_wholesale_cost@8, ss_net_profit@9] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, ss_hdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= 100.00 AND ss_sales_price@0 <= 150.00 AND hd_dep_count@3 = 3 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= 50.00 AND ss_sales_price@0 <= 100.00 AND hd_dep_count@3 = 1 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= 150.00 AND ss_sales_price@0 <= 200.00 AND hd_dep_count@3 = 1, projection=[ss_sold_date_sk@2, ss_addr_sk@4, ss_quantity@5, ss_ext_sales_price@7, ss_ext_wholesale_cost@8, ss_net_profit@9] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),7,2 AND ss_sales_price@0 <= Some(15000),7,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),7,2 AND ss_sales_price@0 <= Some(10000),7,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),7,2 AND ss_sales_price@0 <= Some(20000),7,2, projection=[ss_sold_date_sk@3, ss_hdemo_sk@5, ss_addr_sk@6, ss_quantity@7, ss_sales_price@8, ss_ext_sales_price@9, ss_ext_wholesale_cost@10, ss_net_profit@11, cd_marital_status@1, cd_education_status@2] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= 100.00 AND ss_sales_price@0 <= 150.00 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= 50.00 AND ss_sales_price@0 <= 100.00 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= 150.00 AND ss_sales_price@0 <= 200.00, projection=[ss_sold_date_sk@3, ss_hdemo_sk@5, ss_addr_sk@6, ss_quantity@7, ss_sales_price@8, ss_ext_sales_price@9, ss_ext_wholesale_cost@10, ss_net_profit@11, cd_marital_status@1, cd_education_status@2] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_country@2 = United States AND ca_state@1 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@1 = TX OR ca_state@1 = OH OR ca_state@1 = OR OR ca_state@1 = NM OR ca_state@1 = KY OR ca_state@1 = VA OR ca_state@1 = TX OR ca_state@1 = MS), projection=[ca_address_sk@0, ca_state@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]) AND (ca_state@8 = TX OR ca_state@8 = OH OR ca_state@8 = OR OR ca_state@8 = NM OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = TX OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1633,19 +1600,19 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@4)], projection=[ss_sold_date_sk@1, ss_cdemo_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_quantity@6, ss_sales_price@7, ss_ext_sales_price@8, ss_ext_wholesale_cost@9, ss_net_profit@10] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ FilterExec: (ss_net_profit@9 >= Some(10000),7,2 AND ss_net_profit@9 <= Some(20000),7,2 OR ss_net_profit@9 >= Some(15000),7,2 AND ss_net_profit@9 <= Some(30000),7,2 OR ss_net_profit@9 >= Some(5000),7,2 AND ss_net_profit@9 <= Some(25000),7,2) AND (ss_sales_price@6 >= Some(10000),7,2 AND ss_sales_price@6 <= Some(15000),7,2 OR ss_sales_price@6 >= Some(5000),7,2 AND ss_sales_price@6 <= Some(10000),7,2 OR ss_sales_price@6 >= Some(15000),7,2 AND ss_sales_price@6 <= Some(20000),7,2) + │ FilterExec: (ss_net_profit@9 >= 100.00 AND ss_net_profit@9 <= 200.00 OR ss_net_profit@9 >= 150.00 AND ss_net_profit@9 <= 300.00 OR ss_net_profit@9 >= 50.00 AND ss_net_profit@9 <= 250.00) AND (ss_sales_price@6 >= 100.00 AND ss_sales_price@6 <= 150.00 OR ss_sales_price@6 >= 50.00 AND ss_sales_price@6 <= 100.00 OR ss_sales_price@6 >= 150.00 AND ss_sales_price@6 <= 200.00) │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(10000),7,2 AND ss_net_profit@22 <= Some(20000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(30000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(25000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(10000),7,2 AND ss_net_profit@22 <= Some(20000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(30000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(25000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(10000),7,2 AND ss_net_profit@22 <= Some(20000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(30000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(25000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(10000),7,2 AND ss_net_profit@22 <= Some(20000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(30000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(25000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 100.00 AND ss_net_profit@22 <= 200.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 300.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 250.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 100.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 200.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 300.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 250.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 100.00 AND ss_net_profit@22 <= 200.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 300.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 250.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 100.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 200.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 300.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 250.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 100.00 AND ss_net_profit@22 <= 200.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 300.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 250.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 100.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 200.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 300.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 250.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 100.00 AND ss_net_profit@22 <= 200.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 300.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 250.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 100.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 200.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 300.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 250.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -1661,15 +1628,15 @@ mod tests { #[tokio::test] async fn test_tpcds_14() -> Result<()> { let display = test_tpcds_query("q14").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], preserve_partitioning=[true] │ ProjectionExec: expr=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(y.sales)@5 as sum_sales, sum(y.number_sales)@6 as sum_number_sales] │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, __grouping_id@4 as __grouping_id], aggr=[sum(y.sales), sum(y.number_sales)] - │ [Stage 58] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 61] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 58 ── tasks=2, partitions=3 + ┌───── Stage 61 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([channel@0, i_brand_id@1, i_class_id@2, i_category_id@3, __grouping_id@4], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id)], aggr=[sum(y.sales), sum(y.number_sales)] │ InterleaveExec @@ -1679,21 +1646,21 @@ mod tests { │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=2, input_tasks=1 │ ProjectionExec: expr=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(store_sales.ss_quantity * store_sales.ss_list_price)@3 as sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))@4 as count(Int64(1)), CAST(sum(store_sales.ss_quantity * store_sales.ss_list_price)@3 AS Decimal128(32, 6)) as join_proj_push_down_1] │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] - │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[catalog as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@4 as sales, count(Int64(1))@5 as number_sales] │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_2@1 > avg_sales.average_sales@0, projection=[avg_sales.average_sales@0, i_brand_id@1, i_class_id@2, i_category_id@3, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@4, count(Int64(1))@5] │ CoalescePartitionsExec - │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=2, input_tasks=1 + │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=2, input_tasks=1 │ ProjectionExec: expr=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 as sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))@4 as count(Int64(1)), CAST(sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 AS Decimal128(32, 6)) as join_proj_push_down_2] │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] - │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 40] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[web as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@4 as sales, count(Int64(1))@5 as number_sales] │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_3@1 > avg_sales.average_sales@0, projection=[avg_sales.average_sales@0, i_brand_id@1, i_class_id@2, i_category_id@3, sum(web_sales.ws_quantity * web_sales.ws_list_price)@4, count(Int64(1))@5] │ CoalescePartitionsExec - │ [Stage 43] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=2, input_tasks=1 + │ [Stage 45] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=2, input_tasks=1 │ ProjectionExec: expr=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 as sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))@4 as count(Int64(1)), CAST(sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 AS Decimal128(32, 6)) as join_proj_push_down_3] │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] - │ [Stage 57] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 60] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=1, partitions=2 │ BroadcastExec: input_partitions=1, consumer_tasks=2, output_partitions=2 @@ -1710,638 +1677,626 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_quantity@2, cs_list_price@3] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_quantity@2, ws_list_price@3] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 19 ── tasks=4, partitions=6 + ┌───── Stage 20 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_quantity@3, ss_list_price@4, i_brand_id@5, i_class_id@6, i_category_id@7] │ CoalescePartitionsExec - │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@6, ss_list_price@7, i_brand_id@1, i_class_id@2, i_category_id@3] │ CoalescePartitionsExec - │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=4, partitions=48 + ┌───── Stage 17 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@1, i_class_id@2, i_category_id@3], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 16 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 17 ── tasks=4, partitions=48 + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 18 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 18 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 19 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 24 ── tasks=1, partitions=2 + ┌───── Stage 25 ── tasks=1, partitions=2 │ BroadcastExec: input_partitions=1, consumer_tasks=2, output_partitions=2 │ ProjectionExec: expr=[CAST(avg(sq2.quantity * sq2.list_price)@0 AS Decimal128(32, 6)) as avg_sales.average_sales] │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ CoalescePartitionsExec - │ [Stage 23] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 24] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 23 ── tasks=4, partitions=12 + ┌───── Stage 24 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_list_price@3] │ CoalescePartitionsExec - │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_quantity@2, cs_list_price@3] │ CoalescePartitionsExec - │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_quantity@2, ws_list_price@3] │ CoalescePartitionsExec - │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 + │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 20 ── tasks=4, partitions=24 + ┌───── Stage 21 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 21 ── tasks=4, partitions=12 + ┌───── Stage 22 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 22 ── tasks=4, partitions=12 + ┌───── Stage 23 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 38 ── tasks=4, partitions=6 + ┌───── Stage 40 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_item_sk@0, cs_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 35] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 37] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_item_sk@2, cs_quantity@3, cs_list_price@4, i_brand_id@5, i_class_id@6, i_category_id@7] │ CoalescePartitionsExec - │ [Stage 36] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 38] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7, i_brand_id@1, i_class_id@2, i_category_id@3] │ CoalescePartitionsExec - │ [Stage 37] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 39] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 35 ── tasks=4, partitions=48 + ┌───── Stage 37 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] - │ CoalescePartitionsExec - │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 28] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 31] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ [Stage 34] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 25 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 36] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 26 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@1, i_class_id@2, i_category_id@3], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 28 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 26] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 26 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 27 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 31 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 30] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 29 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 36 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 35] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 29 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 28] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 30 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 34 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 32] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 33] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 32 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 27 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 28 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 32 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 30] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 31] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 33 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 30 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 35 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 33] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 34] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 36 ── tasks=4, partitions=48 + ┌───── Stage 33 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 34 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 38 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 37 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 39 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 43 ── tasks=1, partitions=2 + ┌───── Stage 45 ── tasks=1, partitions=2 │ BroadcastExec: input_partitions=1, consumer_tasks=2, output_partitions=2 │ ProjectionExec: expr=[CAST(avg(sq2.quantity * sq2.list_price)@0 AS Decimal128(32, 6)) as avg_sales.average_sales] │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ CoalescePartitionsExec - │ [Stage 42] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 44] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 42 ── tasks=4, partitions=12 + ┌───── Stage 44 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_list_price@3] │ CoalescePartitionsExec - │ [Stage 39] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 + │ [Stage 41] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_quantity@2, cs_list_price@3] │ CoalescePartitionsExec - │ [Stage 40] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 + │ [Stage 42] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_quantity@2, ws_list_price@3] │ CoalescePartitionsExec - │ [Stage 41] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 + │ [Stage 43] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 39 ── tasks=4, partitions=24 + ┌───── Stage 41 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 40 ── tasks=4, partitions=12 + ┌───── Stage 42 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 41 ── tasks=4, partitions=12 + ┌───── Stage 43 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 57 ── tasks=4, partitions=6 + ┌───── Stage 60 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_item_sk@0, ws_item_sk@0)], projection=[ws_quantity@1, ws_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 54] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 57] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_quantity@3, ws_list_price@4, i_brand_id@5, i_class_id@6, i_category_id@7] │ CoalescePartitionsExec - │ [Stage 55] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 58] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@6, ws_list_price@7, i_brand_id@1, i_class_id@2, i_category_id@3] │ CoalescePartitionsExec - │ [Stage 56] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 59] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 54 ── tasks=4, partitions=48 + ┌───── Stage 57 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] - │ CoalescePartitionsExec - │ [Stage 44] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 47] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true - │ [Stage 50] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ [Stage 53] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 44 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_brand_id@1, brand_id@0), (i_class_id@2, class_id@1), (i_category_id@3, category_id@2)], projection=[i_item_sk@0] + │ [Stage 46] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 56] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 46 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@1, i_class_id@2, i_category_id@3], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 47 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 45] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 46] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 45 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 46 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 50 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 48] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 49] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 48 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 56 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 49] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ [Stage 52] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 55] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 49 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 47] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 48] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 49 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 53 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] - │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@2, i_class_id@3, i_category_id@4] - │ CoalescePartitionsExec - │ [Stage 51] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] - │ CoalescePartitionsExec - │ [Stage 52] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 51 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 47 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 48 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 52 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 50] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 51] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 52 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 50 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 51 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 55 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 12), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 53] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, i_brand_id@1, i_class_id@2, i_category_id@3] + │ CoalescePartitionsExec + │ [Stage 54] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 55 ── tasks=4, partitions=48 + ┌───── Stage 53 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 54 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 58 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 56 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 59 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── "); Ok(()) @@ -2349,7 +2304,7 @@ mod tests { #[tokio::test] async fn test_tpcds_15() -> Result<()> { let display = test_tpcds_query("q15").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ca_zip@0 ASC], fetch=100 │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -2365,43 +2320,42 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_sales_price@2, ca_zip@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], filter=substr(ca_zip@2, 1, 5) IN (SET) ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR ca_state@1 = CA OR ca_state@1 = WA OR ca_state@1 = GA OR cs_sales_price@0 > Some(50000),7,2, projection=[cs_sold_date_sk@3, cs_sales_price@4, ca_zip@2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], filter=substr(ca_zip@2, 1, 5) IN (SET) ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR ca_state@1 = CA OR ca_state@1 = WA OR ca_state@1 = GA OR cs_sales_price@0 > 500.00, projection=[cs_sold_date_sk@3, cs_sales_price@4, ca_zip@2] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@1)], projection=[cs_sold_date_sk@2, cs_sales_price@4, c_current_addr_sk@1] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -2447,10 +2401,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -2465,38 +2419,36 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_state@1 = GA, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2002-02-01 AND d_date@1 <= 2002-04-02, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cr_order_number@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -2504,7 +2456,7 @@ mod tests { #[tokio::test] async fn test_tpcds_17() -> Result<()> { let display = test_tpcds_query("q17").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], fetch=100 │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -2521,7 +2473,7 @@ mod tests { │ ProjectionExec: expr=[CAST(ss_quantity@0 AS Float64) as __common_expr_1, CAST(sr_return_quantity@1 AS Float64) as __common_expr_2, CAST(cs_quantity@2 AS Float64) as __common_expr_3, ss_quantity@0 as ss_quantity, sr_return_quantity@1 as sr_return_quantity, cs_quantity@2 as cs_quantity, s_state@3 as s_state, i_item_id@4 as i_item_id, i_item_desc@5 as i_item_desc] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@4, sr_return_quantity@5, cs_quantity@6, s_state@7, i_item_id@1, i_item_desc@2] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, sr_return_quantity@5, cs_quantity@6, s_state@1] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -2538,13 +2490,13 @@ mod tests { │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -2557,40 +2509,37 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_quarter_name@1 = 2001Q1, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 12), input_partitions=3 @@ -2601,18 +2550,18 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -2637,12 +2586,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, cs_sales_price@3, cs_coupon_amt@4, cs_net_profit@5, cd_dep_count@6, c_birth_year@7, ca_county@8, ca_state@9, ca_country@10, i_item_id@12] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -2659,22 +2607,20 @@ mod tests { ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_state@2 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]) - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_current_cdemo_sk@8], 12), input_partitions=3 @@ -2688,12 +2634,11 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: c_birth_month@3 IN (SET) ([1, 6, 8, 9, 12, 2]), projection=[c_customer_sk@0, c_current_cdemo_sk@1, c_current_addr_sk@2, c_birth_year@4] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -2702,23 +2647,23 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -2726,16 +2671,17 @@ mod tests { #[tokio::test] async fn test_tpcds_19() -> Result<()> { let display = test_tpcds_query("q19").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ext_price@4 DESC, brand@1 ASC NULLS LAST, brand_id@0 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], fetch=100 │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[ext_price@4 DESC, brand@1 ASC NULLS LAST, brand_id@0 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact, sum(store_sales.ss_ext_sales_price)@4 as ext_price] - │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(store_sales.ss_ext_sales_price)@4 DESC, i_brand@0 ASC NULLS LAST, i_brand_id@1 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1, i_manufact_id@2, i_manufact@3], 6), input_partitions=3 @@ -2746,12 +2692,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@6, ca_address_sk@0)], projection=[ss_store_sk@0, ss_ext_sales_price@1, i_brand_id@2, i_brand@3, i_manufact_id@4, i_manufact@5, ca_zip@8] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -2766,12 +2711,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, c_customer_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, i_brand_id@3, i_brand@4, i_manufact_id@5, i_manufact@6, c_current_addr_sk@8] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -2782,30 +2726,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manager_id@5 = 8, projection=[i_item_sk@0, i_brand_id@1, i_brand@2, i_manufact_id@3, i_manufact@4] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1998, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] └────────────────────────────────────────────────── "); Ok(()) @@ -2837,30 +2779,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] └────────────────────────────────────────────────── "#); Ok(()) @@ -2895,30 +2835,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@2 >= Some(99),7,2 AND i_current_price@2 <= Some(149),7,2, projection=[i_item_sk@0, i_item_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] + │ FilterExec: i_current_price@2 >= 0.99 AND i_current_price@2 <= 1.49, projection=[i_item_sk@0, i_item_id@1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -2934,7 +2872,7 @@ mod tests { #[tokio::test] async fn test_tpcds_22() -> Result<()> { let display = test_tpcds_query("q22").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], fetch=100 │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -2950,33 +2888,32 @@ mod tests { │ AggregateExec: mode=Partial, gby=[(NULL as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category)], aggr=[avg(inventory.inv_quantity_on_hand)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@0)], projection=[inv_quantity_on_hand@6, i_brand@1, i_class@2, i_category@3, i_product_name@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[inv_item_sk@2, inv_quantity_on_hand@3] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -2984,33 +2921,30 @@ mod tests { #[tokio::test] async fn test_tpcds_23() -> Result<()> { let display = test_tpcds_query("q23").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], fetch=100 - │ [Stage 31] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 31 ── tasks=2, partitions=3 + ┌───── Stage 33 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], preserve_partitioning=[true] │ InterleaveExec │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@2 as sales] │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] - │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=4, partitions=6 + ┌───── Stage 16 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_customer_sk@0, c_customer_sk@0)], projection=[cs_quantity@1, cs_list_price@2, c_first_name@3, c_last_name@4] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(cs_bill_customer_sk@0, c_customer_sk@0)], projection=[cs_quantity@1, cs_list_price@2, c_first_name@3, c_last_name@4] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 7 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, item_sk@0)], projection=[cs_bill_customer_sk@0, cs_quantity@2, cs_list_price@3, c_first_name@4, c_last_name@5] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3027,30 +2961,29 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@1)], projection=[cs_sold_date_sk@3, cs_bill_customer_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7, c_first_name@1, c_last_name@2] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 12), input_partitions=3 @@ -3062,266 +2995,266 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=12 + ┌───── Stage 15 ── tasks=2, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] - │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] - │ CrossJoinExec - │ CoalescePartitionsExec - │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] - │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=1, partitions=4 - │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 - │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] - │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] - │ CoalescePartitionsExec - │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=2, partitions=3 - │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] - │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 14 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 12 ── tasks=1, partitions=4 + │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] - │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 11 ── tasks=2, partitions=3 + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + ┌───── Stage 10 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 30 ── tasks=4, partitions=6 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 32 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_customer_sk@0, c_customer_sk@0)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] - │ CoalescePartitionsExec - │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] - │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_bill_customer_sk@0, c_customer_sk@0)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 31] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 22 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 23 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, item_sk@0)], projection=[ws_bill_customer_sk@1, ws_quantity@2, ws_list_price@3, c_first_name@4, c_last_name@5] │ CoalescePartitionsExec - │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[i_item_sk@0 as item_sk] │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] │ AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] - │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 18 ── tasks=4, partitions=48 + ┌───── Stage 19 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_bill_customer_sk@3, ws_quantity@4, ws_list_price@5, c_first_name@6, c_last_name@7] │ CoalescePartitionsExec - │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@2)], projection=[ws_sold_date_sk@3, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_list_price@7, c_first_name@1, c_last_name@2] │ CoalescePartitionsExec - │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=4, partitions=48 + ┌───── Stage 17 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 17 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 18 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 21 ── tasks=4, partitions=12 + ┌───── Stage 22 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 12), input_partitions=3 │ AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@1, ss_item_sk@0)], projection=[d_date@3, itemdesc@0, i_item_sk@1] │ CoalescePartitionsExec - │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, d_date@1] │ CoalescePartitionsExec - │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 19 ── tasks=4, partitions=48 + ┌───── Stage 20 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[substr(i_item_desc@4, 1, 30) as itemdesc, i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 20 ── tasks=4, partitions=48 + ┌───── Stage 21 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] └────────────────────────────────────────────────── - ┌───── Stage 29 ── tasks=4, partitions=12 + ┌───── Stage 31 ── tasks=2, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] - │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] - │ CrossJoinExec - │ CoalescePartitionsExec - │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] - │ CoalescePartitionsExec - │ [Stage 28] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 27 ── tasks=1, partitions=4 - │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 - │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] - │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] - │ CoalescePartitionsExec - │ [Stage 26] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 26 ── tasks=2, partitions=3 - │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] - │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] - │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] - │ [Stage 25] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ FilterExec: CAST(sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 AS Decimal128(38, 15)) > CAST(0.5 * CAST(max(max_store_sales.tpcds_cmax)@2 AS Float64) AS Decimal128(38, 15)), projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 30 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ CoalescePartitionsExec + │ [Stage 28] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@0] + │ CoalescePartitionsExec + │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 28 ── tasks=1, partitions=4 + │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ [Stage 27] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 25 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] - │ CoalescePartitionsExec - │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] - │ CoalescePartitionsExec - │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 27 ── tasks=2, partitions=3 + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 23 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] - └────────────────────────────────────────────────── - ┌───── Stage 24 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + ┌───── Stage 26 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2, ss_sales_price@3, c_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_sold_date_sk@1, ss_quantity@3, ss_sales_price@4, c_customer_sk@0] + │ CoalescePartitionsExec + │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 28 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet - └────────────────────────────────────────────────── + ┌───── Stage 24 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 25 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 29 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_24() -> Result<()> { let display = test_tpcds_query("q24").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC NULLS LAST, c_first_name@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST] │ SortExec: expr=[c_last_name@0 ASC NULLS LAST, c_first_name@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST], preserve_partitioning=[true] @@ -3348,13 +3281,13 @@ mod tests { │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@9), (ca_zip@2, s_zip@3)], filter=c_birth_country@0 != upper(ca_country@1), projection=[ss_net_paid@4, s_store_name@5, s_state@6, i_current_price@8, i_size@9, i_color@10, i_units@11, i_manager_id@12, c_first_name@14, c_last_name@15, ca_state@1] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_net_paid@6, s_store_name@7, s_state@8, s_zip@9, i_current_price@10, i_size@11, i_color@12, i_units@13, i_manager_id@14, c_current_addr_sk@1, c_first_name@2, c_last_name@3, c_birth_country@4] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_customer_sk@7, ss_net_paid@8, s_store_name@9, s_state@10, s_zip@11, i_current_price@1, i_size@2, i_color@3, i_units@4, i_manager_id@5] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@2)], projection=[ss_item_sk@4, ss_customer_sk@5, ss_net_paid@7, s_store_name@1, s_state@2, s_zip@3] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3362,29 +3295,29 @@ mod tests { │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -3407,10 +3340,10 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2], 3), input_partitions=3 @@ -3424,10 +3357,10 @@ mod tests { │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([5]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@9), (ca_zip@2, s_zip@3)], filter=c_birth_country@0 != upper(ca_country@1), projection=[ss_net_paid@4, s_store_name@5, s_state@6, i_current_price@8, i_size@9, i_color@10, i_units@11, i_manager_id@12, c_first_name@14, c_last_name@15, ca_state@1] │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[ss_net_paid@6, s_store_name@7, s_state@8, s_zip@9, i_current_price@10, i_size@11, i_color@12, i_units@13, i_manager_id@14, c_current_addr_sk@1, c_first_name@2, c_last_name@3, c_birth_country@4] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_customer_sk@7, ss_net_paid@8, s_store_name@9, s_state@10, s_zip@11, i_current_price@1, i_size@2, i_color@3, i_units@4, i_manager_id@5] │ CoalescePartitionsExec │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3438,31 +3371,30 @@ mod tests { │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 9 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_color@3 = peach - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -3485,10 +3417,10 @@ mod tests { ┌───── Stage 14 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -3496,131 +3428,130 @@ mod tests { #[tokio::test] async fn test_tpcds_25() -> Result<()> { let display = test_tpcds_query("q25").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_net_profit)@4 as store_sales_profit, sum(store_returns.sr_net_loss)@5 as store_returns_loss, sum(catalog_sales.cs_net_profit)@6 as catalog_sales_profit] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_net_profit@4, sr_net_loss@5, cs_net_profit@6, s_store_id@7, s_store_name@8, i_item_id@1, i_item_desc@2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_net_profit@1, sr_net_loss@2, cs_net_profit@3, s_store_id@4, s_store_name@5, i_item_id@7, i_item_desc@8] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@3, ss_net_profit@5, sr_net_loss@6, cs_net_profit@7, s_store_id@1, s_store_name@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@4)], projection=[ss_item_sk@1, ss_store_sk@2, ss_net_profit@3, sr_net_loss@4, cs_net_profit@6] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sr_returned_date_sk@3)], projection=[ss_item_sk@1, ss_store_sk@2, ss_net_profit@3, sr_net_loss@5, cs_sold_date_sk@6, cs_net_profit@7] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_store_sk@3, ss_net_profit@4, sr_returned_date_sk@5, sr_net_loss@6, cs_sold_date_sk@7, cs_net_profit@8] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_net_profit@7, sr_returned_date_sk@8, sr_net_loss@11, cs_sold_date_sk@0, cs_net_profit@3] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=48 + ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_moy@2 = 4 AND d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_net_profit@10, sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_net_loss@4] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@3, ss_net_profit@5, sr_net_loss@6, cs_net_profit@7, s_store_id@1, s_store_name@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@4)], projection=[ss_item_sk@1, ss_store_sk@2, ss_net_profit@3, sr_net_loss@4, cs_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sr_returned_date_sk@3)], projection=[ss_item_sk@1, ss_store_sk@2, ss_net_profit@3, sr_net_loss@5, cs_sold_date_sk@6, cs_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_store_sk@3, ss_net_profit@4, sr_returned_date_sk@5, sr_net_loss@6, cs_sold_date_sk@7, cs_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_net_profit@7, sr_returned_date_sk@8, sr_net_loss@11, cs_sold_date_sk@0, cs_net_profit@3] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 12), input_partitions=3 + ┌───── Stage 1 ── tasks=4, partitions=32 + │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 12), input_partitions=3 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_moy@2 = 4 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 8 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 12), input_partitions=3 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_net_profit@10, sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_net_loss@4] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 7 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_26() -> Result<()> { let display = test_tpcds_query("q26").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(catalog_sales.cs_quantity)@1 as agg1, avg(catalog_sales.cs_list_price)@2 as agg2, avg(catalog_sales.cs_coupon_amt)@3 as agg3, avg(catalog_sales.cs_sales_price)@4 as agg4] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -3630,7 +3561,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_promo_sk@3, cs_quantity@4, cs_list_price@5, cs_sales_price@6, cs_coupon_amt@7, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_item_sk@2, cs_promo_sk@3, cs_quantity@4, cs_list_price@5, cs_sales_price@6, cs_coupon_amt@7] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3648,23 +3579,22 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:..]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet:..]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -3673,15 +3603,15 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -3689,7 +3619,7 @@ mod tests { #[tokio::test] async fn test_tpcds_27() -> Result<()> { let display = test_tpcds_query("q27").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC, s_state@1 ASC], fetch=100 │ [Stage 19] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -3714,7 +3644,7 @@ mod tests { │ ProjectionExec: expr=[i_item_id@0 as i_item_id, s_state@1 as s_state, ss_quantity@2 as agg1, ss_list_price@3 as agg2, ss_coupon_amt@4 as agg3, ss_sales_price@5 as agg4] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, s_state@7, ss_quantity@3, ss_list_price@4, ss_coupon_amt@6, ss_sales_price@5] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7, s_state@1] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3725,13 +3655,13 @@ mod tests { │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -3746,12 +3676,11 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -3760,15 +3689,15 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 @@ -3776,7 +3705,7 @@ mod tests { │ ProjectionExec: expr=[i_item_id@0 as i_item_id, ss_quantity@1 as agg1, ss_list_price@2 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@4 as agg4] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, ss_quantity@3, ss_list_price@4, ss_coupon_amt@6, ss_sales_price@5] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@1, ss_quantity@3, ss_list_price@4, ss_sales_price@5, ss_coupon_amt@6] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3787,13 +3716,13 @@ mod tests { │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -3808,12 +3737,11 @@ mod tests { ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -3822,22 +3750,22 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 18 ── tasks=4, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] │ ProjectionExec: expr=[ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@2 as agg3, ss_sales_price@3 as agg4] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@2, ss_list_price@3, ss_coupon_amt@5, ss_sales_price@4] │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@1, ss_quantity@3, ss_list_price@4, ss_sales_price@5, ss_coupon_amt@6] │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -3848,50 +3776,49 @@ mod tests { │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -3958,57 +3885,57 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),7,2 AND ss_list_price@2 <= Some(16400),7,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),7,2 AND ss_wholesale_cost@1 <= Some(2700),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= 154.00 AND ss_list_price@2 <= 164.00 OR ss_coupon_amt@3 >= 7326.00 AND ss_coupon_amt@3 <= 8326.00 OR ss_wholesale_cost@1 >= 7.00 AND ss_wholesale_cost@1 <= 27.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= Some(15400),7,2 AND ss_list_price@12 <= Some(16400),7,2 OR ss_coupon_amt@19 >= Some(732600),7,2 AND ss_coupon_amt@19 <= Some(832600),7,2 OR ss_wholesale_cost@11 >= Some(700),7,2 AND ss_wholesale_cost@11 <= Some(2700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= Some(15400),7,2 AND ss_list_price@12 <= Some(16400),7,2 OR ss_coupon_amt@19 >= Some(732600),7,2 AND ss_coupon_amt@19 <= Some(832600),7,2 OR ss_wholesale_cost@11 >= Some(700),7,2 AND ss_wholesale_cost@11 <= Some(2700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= Some(15400),7,2 AND ss_list_price@12 <= Some(16400),7,2 OR ss_coupon_amt@19 >= Some(732600),7,2 AND ss_coupon_amt@19 <= Some(832600),7,2 OR ss_wholesale_cost@11 >= Some(700),7,2 AND ss_wholesale_cost@11 <= Some(2700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= Some(15400),7,2 AND ss_list_price@12 <= Some(16400),7,2 OR ss_coupon_amt@19 >= Some(732600),7,2 AND ss_coupon_amt@19 <= Some(832600),7,2 OR ss_wholesale_cost@11 >= Some(700),7,2 AND ss_wholesale_cost@11 <= Some(2700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= 154.00 AND ss_list_price@12 <= 164.00 OR ss_coupon_amt@19 >= 7326.00 AND ss_coupon_amt@19 <= 8326.00 OR ss_wholesale_cost@11 >= 7.00 AND ss_wholesale_cost@11 <= 27.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 154.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 164.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 7326.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 8326.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 7.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 27.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= 154.00 AND ss_list_price@12 <= 164.00 OR ss_coupon_amt@19 >= 7326.00 AND ss_coupon_amt@19 <= 8326.00 OR ss_wholesale_cost@11 >= 7.00 AND ss_wholesale_cost@11 <= 27.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 154.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 164.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 7326.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 8326.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 7.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 27.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= 154.00 AND ss_list_price@12 <= 164.00 OR ss_coupon_amt@19 >= 7326.00 AND ss_coupon_amt@19 <= 8326.00 OR ss_wholesale_cost@11 >= 7.00 AND ss_wholesale_cost@11 <= 27.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 154.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 164.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 7326.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 8326.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 7.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 27.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= 154.00 AND ss_list_price@12 <= 164.00 OR ss_coupon_amt@19 >= 7326.00 AND ss_coupon_amt@19 <= 8326.00 OR ss_wholesale_cost@11 >= 7.00 AND ss_wholesale_cost@11 <= 27.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 154.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 164.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 7326.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 8326.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 7.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 27.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),7,2 AND ss_list_price@2 <= Some(13200),7,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),7,2 AND ss_wholesale_cost@1 <= Some(3700),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= 122.00 AND ss_list_price@2 <= 132.00 OR ss_coupon_amt@3 >= 836.00 AND ss_coupon_amt@3 <= 1836.00 OR ss_wholesale_cost@1 >= 17.00 AND ss_wholesale_cost@1 <= 37.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= Some(12200),7,2 AND ss_list_price@12 <= Some(13200),7,2 OR ss_coupon_amt@19 >= Some(83600),7,2 AND ss_coupon_amt@19 <= Some(183600),7,2 OR ss_wholesale_cost@11 >= Some(1700),7,2 AND ss_wholesale_cost@11 <= Some(3700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= Some(12200),7,2 AND ss_list_price@12 <= Some(13200),7,2 OR ss_coupon_amt@19 >= Some(83600),7,2 AND ss_coupon_amt@19 <= Some(183600),7,2 OR ss_wholesale_cost@11 >= Some(1700),7,2 AND ss_wholesale_cost@11 <= Some(3700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= Some(12200),7,2 AND ss_list_price@12 <= Some(13200),7,2 OR ss_coupon_amt@19 >= Some(83600),7,2 AND ss_coupon_amt@19 <= Some(183600),7,2 OR ss_wholesale_cost@11 >= Some(1700),7,2 AND ss_wholesale_cost@11 <= Some(3700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= Some(12200),7,2 AND ss_list_price@12 <= Some(13200),7,2 OR ss_coupon_amt@19 >= Some(83600),7,2 AND ss_coupon_amt@19 <= Some(183600),7,2 OR ss_wholesale_cost@11 >= Some(1700),7,2 AND ss_wholesale_cost@11 <= Some(3700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= 122.00 AND ss_list_price@12 <= 132.00 OR ss_coupon_amt@19 >= 836.00 AND ss_coupon_amt@19 <= 1836.00 OR ss_wholesale_cost@11 >= 17.00 AND ss_wholesale_cost@11 <= 37.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 122.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 132.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 836.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1836.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 17.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 37.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= 122.00 AND ss_list_price@12 <= 132.00 OR ss_coupon_amt@19 >= 836.00 AND ss_coupon_amt@19 <= 1836.00 OR ss_wholesale_cost@11 >= 17.00 AND ss_wholesale_cost@11 <= 37.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 122.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 132.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 836.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1836.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 17.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 37.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= 122.00 AND ss_list_price@12 <= 132.00 OR ss_coupon_amt@19 >= 836.00 AND ss_coupon_amt@19 <= 1836.00 OR ss_wholesale_cost@11 >= 17.00 AND ss_wholesale_cost@11 <= 37.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 122.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 132.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 836.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1836.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 17.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 37.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= 122.00 AND ss_list_price@12 <= 132.00 OR ss_coupon_amt@19 >= 836.00 AND ss_coupon_amt@19 <= 1836.00 OR ss_wholesale_cost@11 >= 17.00 AND ss_wholesale_cost@11 <= 37.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 122.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 132.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 836.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1836.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 17.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 37.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),7,2 AND ss_list_price@2 <= Some(14500),7,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),7,2 AND ss_wholesale_cost@1 <= Some(5800),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= 135.00 AND ss_list_price@2 <= 145.00 OR ss_coupon_amt@3 >= 6071.00 AND ss_coupon_amt@3 <= 7071.00 OR ss_wholesale_cost@1 >= 38.00 AND ss_wholesale_cost@1 <= 58.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= Some(13500),7,2 AND ss_list_price@12 <= Some(14500),7,2 OR ss_coupon_amt@19 >= Some(607100),7,2 AND ss_coupon_amt@19 <= Some(707100),7,2 OR ss_wholesale_cost@11 >= Some(3800),7,2 AND ss_wholesale_cost@11 <= Some(5800),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= Some(13500),7,2 AND ss_list_price@12 <= Some(14500),7,2 OR ss_coupon_amt@19 >= Some(607100),7,2 AND ss_coupon_amt@19 <= Some(707100),7,2 OR ss_wholesale_cost@11 >= Some(3800),7,2 AND ss_wholesale_cost@11 <= Some(5800),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= Some(13500),7,2 AND ss_list_price@12 <= Some(14500),7,2 OR ss_coupon_amt@19 >= Some(607100),7,2 AND ss_coupon_amt@19 <= Some(707100),7,2 OR ss_wholesale_cost@11 >= Some(3800),7,2 AND ss_wholesale_cost@11 <= Some(5800),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= Some(13500),7,2 AND ss_list_price@12 <= Some(14500),7,2 OR ss_coupon_amt@19 >= Some(607100),7,2 AND ss_coupon_amt@19 <= Some(707100),7,2 OR ss_wholesale_cost@11 >= Some(3800),7,2 AND ss_wholesale_cost@11 <= Some(5800),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= 135.00 AND ss_list_price@12 <= 145.00 OR ss_coupon_amt@19 >= 6071.00 AND ss_coupon_amt@19 <= 7071.00 OR ss_wholesale_cost@11 >= 38.00 AND ss_wholesale_cost@11 <= 58.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 135.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 145.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 6071.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 7071.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 38.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 58.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= 135.00 AND ss_list_price@12 <= 145.00 OR ss_coupon_amt@19 >= 6071.00 AND ss_coupon_amt@19 <= 7071.00 OR ss_wholesale_cost@11 >= 38.00 AND ss_wholesale_cost@11 <= 58.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 135.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 145.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 6071.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 7071.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 38.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 58.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= 135.00 AND ss_list_price@12 <= 145.00 OR ss_coupon_amt@19 >= 6071.00 AND ss_coupon_amt@19 <= 7071.00 OR ss_wholesale_cost@11 >= 38.00 AND ss_wholesale_cost@11 <= 58.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 135.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 145.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 6071.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 7071.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 38.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 58.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= 135.00 AND ss_list_price@12 <= 145.00 OR ss_coupon_amt@19 >= 6071.00 AND ss_coupon_amt@19 <= 7071.00 OR ss_wholesale_cost@11 >= 38.00 AND ss_wholesale_cost@11 <= 58.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 135.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 145.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 6071.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 7071.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 38.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 58.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),7,2 AND ss_list_price@2 <= Some(15200),7,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),7,2 AND ss_wholesale_cost@1 <= Some(9900),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= 142.00 AND ss_list_price@2 <= 152.00 OR ss_coupon_amt@3 >= 12214.00 AND ss_coupon_amt@3 <= 13214.00 OR ss_wholesale_cost@1 >= 79.00 AND ss_wholesale_cost@1 <= 99.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= Some(14200),7,2 AND ss_list_price@12 <= Some(15200),7,2 OR ss_coupon_amt@19 >= Some(1221400),7,2 AND ss_coupon_amt@19 <= Some(1321400),7,2 OR ss_wholesale_cost@11 >= Some(7900),7,2 AND ss_wholesale_cost@11 <= Some(9900),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= Some(14200),7,2 AND ss_list_price@12 <= Some(15200),7,2 OR ss_coupon_amt@19 >= Some(1221400),7,2 AND ss_coupon_amt@19 <= Some(1321400),7,2 OR ss_wholesale_cost@11 >= Some(7900),7,2 AND ss_wholesale_cost@11 <= Some(9900),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= Some(14200),7,2 AND ss_list_price@12 <= Some(15200),7,2 OR ss_coupon_amt@19 >= Some(1221400),7,2 AND ss_coupon_amt@19 <= Some(1321400),7,2 OR ss_wholesale_cost@11 >= Some(7900),7,2 AND ss_wholesale_cost@11 <= Some(9900),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= Some(14200),7,2 AND ss_list_price@12 <= Some(15200),7,2 OR ss_coupon_amt@19 >= Some(1221400),7,2 AND ss_coupon_amt@19 <= Some(1321400),7,2 OR ss_wholesale_cost@11 >= Some(7900),7,2 AND ss_wholesale_cost@11 <= Some(9900),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= 142.00 AND ss_list_price@12 <= 152.00 OR ss_coupon_amt@19 >= 12214.00 AND ss_coupon_amt@19 <= 13214.00 OR ss_wholesale_cost@11 >= 79.00 AND ss_wholesale_cost@11 <= 99.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 142.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 152.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 12214.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 13214.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 79.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 99.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= 142.00 AND ss_list_price@12 <= 152.00 OR ss_coupon_amt@19 >= 12214.00 AND ss_coupon_amt@19 <= 13214.00 OR ss_wholesale_cost@11 >= 79.00 AND ss_wholesale_cost@11 <= 99.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 142.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 152.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 12214.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 13214.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 79.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 99.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= 142.00 AND ss_list_price@12 <= 152.00 OR ss_coupon_amt@19 >= 12214.00 AND ss_coupon_amt@19 <= 13214.00 OR ss_wholesale_cost@11 >= 79.00 AND ss_wholesale_cost@11 <= 99.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 142.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 152.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 12214.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 13214.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 79.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 99.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= 142.00 AND ss_list_price@12 <= 152.00 OR ss_coupon_amt@19 >= 12214.00 AND ss_coupon_amt@19 <= 13214.00 OR ss_wholesale_cost@11 >= 79.00 AND ss_wholesale_cost@11 <= 99.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 142.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 152.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 12214.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 13214.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 79.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 99.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),7,2 AND ss_list_price@2 <= Some(1800),7,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),7,2 AND ss_wholesale_cost@1 <= Some(7700),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= 8.00 AND ss_list_price@2 <= 18.00 OR ss_coupon_amt@3 >= 459.00 AND ss_coupon_amt@3 <= 1459.00 OR ss_wholesale_cost@1 >= 57.00 AND ss_wholesale_cost@1 <= 77.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= Some(800),7,2 AND ss_list_price@12 <= Some(1800),7,2 OR ss_coupon_amt@19 >= Some(45900),7,2 AND ss_coupon_amt@19 <= Some(145900),7,2 OR ss_wholesale_cost@11 >= Some(5700),7,2 AND ss_wholesale_cost@11 <= Some(7700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= Some(800),7,2 AND ss_list_price@12 <= Some(1800),7,2 OR ss_coupon_amt@19 >= Some(45900),7,2 AND ss_coupon_amt@19 <= Some(145900),7,2 OR ss_wholesale_cost@11 >= Some(5700),7,2 AND ss_wholesale_cost@11 <= Some(7700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= Some(800),7,2 AND ss_list_price@12 <= Some(1800),7,2 OR ss_coupon_amt@19 >= Some(45900),7,2 AND ss_coupon_amt@19 <= Some(145900),7,2 OR ss_wholesale_cost@11 >= Some(5700),7,2 AND ss_wholesale_cost@11 <= Some(7700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= Some(800),7,2 AND ss_list_price@12 <= Some(1800),7,2 OR ss_coupon_amt@19 >= Some(45900),7,2 AND ss_coupon_amt@19 <= Some(145900),7,2 OR ss_wholesale_cost@11 >= Some(5700),7,2 AND ss_wholesale_cost@11 <= Some(7700),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= 8.00 AND ss_list_price@12 <= 18.00 OR ss_coupon_amt@19 >= 459.00 AND ss_coupon_amt@19 <= 1459.00 OR ss_wholesale_cost@11 >= 57.00 AND ss_wholesale_cost@11 <= 77.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 8.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 18.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 459.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1459.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 57.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 77.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= 8.00 AND ss_list_price@12 <= 18.00 OR ss_coupon_amt@19 >= 459.00 AND ss_coupon_amt@19 <= 1459.00 OR ss_wholesale_cost@11 >= 57.00 AND ss_wholesale_cost@11 <= 77.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 8.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 18.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 459.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1459.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 57.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 77.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= 8.00 AND ss_list_price@12 <= 18.00 OR ss_coupon_amt@19 >= 459.00 AND ss_coupon_amt@19 <= 1459.00 OR ss_wholesale_cost@11 >= 57.00 AND ss_wholesale_cost@11 <= 77.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 8.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 18.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 459.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1459.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 57.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 77.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= 8.00 AND ss_list_price@12 <= 18.00 OR ss_coupon_amt@19 >= 459.00 AND ss_coupon_amt@19 <= 1459.00 OR ss_wholesale_cost@11 >= 57.00 AND ss_wholesale_cost@11 <= 77.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 8.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 18.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 459.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 1459.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 57.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 77.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] - │ FilterExec: ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),7,2 AND ss_list_price@2 <= Some(10000),7,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),7,2 AND ss_wholesale_cost@1 <= Some(5100),7,2), projection=[ss_list_price@2] + │ FilterExec: ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= 90.00 AND ss_list_price@2 <= 100.00 OR ss_coupon_amt@3 >= 2323.00 AND ss_coupon_amt@3 <= 3323.00 OR ss_wholesale_cost@1 >= 31.00 AND ss_wholesale_cost@1 <= 51.00), projection=[ss_list_price@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= Some(9000),7,2 AND ss_list_price@12 <= Some(10000),7,2 OR ss_coupon_amt@19 >= Some(232300),7,2 AND ss_coupon_amt@19 <= Some(332300),7,2 OR ss_wholesale_cost@11 >= Some(3100),7,2 AND ss_wholesale_cost@11 <= Some(5100),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= Some(9000),7,2 AND ss_list_price@12 <= Some(10000),7,2 OR ss_coupon_amt@19 >= Some(232300),7,2 AND ss_coupon_amt@19 <= Some(332300),7,2 OR ss_wholesale_cost@11 >= Some(3100),7,2 AND ss_wholesale_cost@11 <= Some(5100),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= Some(9000),7,2 AND ss_list_price@12 <= Some(10000),7,2 OR ss_coupon_amt@19 >= Some(232300),7,2 AND ss_coupon_amt@19 <= Some(332300),7,2 OR ss_wholesale_cost@11 >= Some(3100),7,2 AND ss_wholesale_cost@11 <= Some(5100),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= Some(9000),7,2 AND ss_list_price@12 <= Some(10000),7,2 OR ss_coupon_amt@19 >= Some(232300),7,2 AND ss_coupon_amt@19 <= Some(332300),7,2 OR ss_wholesale_cost@11 >= Some(3100),7,2 AND ss_wholesale_cost@11 <= Some(5100),7,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),7,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),7,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),7,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= 90.00 AND ss_list_price@12 <= 100.00 OR ss_coupon_amt@19 >= 2323.00 AND ss_coupon_amt@19 <= 3323.00 OR ss_wholesale_cost@11 >= 31.00 AND ss_wholesale_cost@11 <= 51.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 90.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 100.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 2323.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 3323.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 31.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 51.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= 90.00 AND ss_list_price@12 <= 100.00 OR ss_coupon_amt@19 >= 2323.00 AND ss_coupon_amt@19 <= 3323.00 OR ss_wholesale_cost@11 >= 31.00 AND ss_wholesale_cost@11 <= 51.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 90.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 100.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 2323.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 3323.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 31.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 51.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= 90.00 AND ss_list_price@12 <= 100.00 OR ss_coupon_amt@19 >= 2323.00 AND ss_coupon_amt@19 <= 3323.00 OR ss_wholesale_cost@11 >= 31.00 AND ss_wholesale_cost@11 <= 51.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 90.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 100.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 2323.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 3323.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 31.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 51.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= 90.00 AND ss_list_price@12 <= 100.00 OR ss_coupon_amt@19 >= 2323.00 AND ss_coupon_amt@19 <= 3323.00 OR ss_wholesale_cost@11 >= 31.00 AND ss_wholesale_cost@11 <= 51.00), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= 90.00 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= 100.00 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= 2323.00 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= 3323.00 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= 31.00 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= 51.00), required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -4016,23 +3943,24 @@ mod tests { #[tokio::test] async fn test_tpcds_29() -> Result<()> { let display = test_tpcds_query("q29").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_quantity)@4 as store_sales_quantity, sum(store_returns.sr_return_quantity)@5 as store_returns_quantity, sum(catalog_sales.cs_quantity)@6 as catalog_sales_quantity] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@4, sr_return_quantity@5, cs_quantity@6, s_store_id@7, s_store_name@8, i_item_id@1, i_item_desc@2] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@3, ss_quantity@5, sr_return_quantity@6, cs_quantity@7, s_store_id@1, s_store_name@2] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -4049,13 +3977,13 @@ mod tests { │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -4068,40 +3996,37 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 >= 9 AND d_moy@2 <= 12 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 9 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 12), input_partitions=3 @@ -4112,18 +4037,18 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -4237,7 +4162,7 @@ mod tests { #[tokio::test] async fn test_tpcds_31() -> Result<()> { let display = test_tpcds_query("q31").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ca_county@0 ASC NULLS LAST] │ [Stage 25] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -4246,7 +4171,7 @@ mod tests { │ SortExec: expr=[ca_county@0 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[ca_county@2 as ca_county, d_year@3 as d_year, __common_expr_1@0 / CAST(web_sales@6 AS Float64) as web_q1_q2_increase, __common_expr_2@1 / CAST(store_sales@4 AS Float64) as store_q1_q2_increase, CAST(web_sales@7 AS Float64) / __common_expr_1@0 as web_q2_q3_increase, CAST(store_sales@5 AS Float64) / __common_expr_2@1 as store_q2_q3_increase] │ ProjectionExec: expr=[CAST(web_sales@0 AS Float64) as __common_expr_1, CAST(store_sales@1 AS Float64) as __common_expr_2, ca_county@2 as ca_county, d_year@3 as d_year, store_sales@4 as store_sales, store_sales@5 as store_sales, web_sales@6 as web_sales, web_sales@7 as web_sales] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[web_sales@7, store_sales@3, ca_county@0, d_year@1, store_sales@2, store_sales@4, web_sales@6, web_sales@9] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > 0.00 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > 0.00 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[web_sales@7, store_sales@3, ca_county@0, d_year@1, store_sales@2, store_sales@4, web_sales@6, web_sales@9] │ CoalescePartitionsExec │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] @@ -4255,7 +4180,7 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 21 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, ca_county@5, web_sales@6, web_sales@8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > 0.00 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > 0.00 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, ca_county@5, web_sales@6, web_sales@8] │ CoalescePartitionsExec │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=1 │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] @@ -4285,12 +4210,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, ca_address_sk@0)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -4298,20 +4222,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([ca_county@0], 3), input_partitions=3 @@ -4324,33 +4247,32 @@ mod tests { │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@0)], projection=[ss_ext_sales_price@3, d_year@4, d_qoy@5, ca_county@1] │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_addr_sk@4, ss_ext_sales_price@5, d_year@1, d_qoy@2] │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 5 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([ca_county@0], 3), input_partitions=3 @@ -4363,33 +4285,32 @@ mod tests { │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@0)], projection=[ss_ext_sales_price@3, d_year@4, d_qoy@5, ca_county@1] │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_addr_sk@4, ss_ext_sales_price@5, d_year@1, d_qoy@2] │ CoalescePartitionsExec │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 9 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([ca_county@0], 3), input_partitions=3 @@ -4402,33 +4323,32 @@ mod tests { │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@0)], projection=[ss_ext_sales_price@3, d_year@4, d_qoy@5, ca_county@1] │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_addr_sk@4, ss_ext_sales_price@5, d_year@1, d_qoy@2] │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 20 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 @@ -4436,12 +4356,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, ca_address_sk@0)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] │ CoalescePartitionsExec │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -4449,20 +4368,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 18 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 24 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 @@ -4470,12 +4388,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, ca_address_sk@0)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] │ CoalescePartitionsExec │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 23 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -4483,20 +4400,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 22 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] └────────────────────────────────────────────────── "); Ok(()) @@ -4514,7 +4430,7 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(catalog_sales.cs_ext_discount_amt)] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@1, cs_item_sk@1)], filter=CAST(cs_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt)@1, projection=[cs_ext_discount_amt@0] + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_item_sk@1, cs_item_sk@1)], filter=CAST(cs_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt)@1, projection=[cs_ext_discount_amt@0] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(catalog_sales.cs_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt), cs_item_sk@0 as cs_item_sk] │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[avg(catalog_sales.cs_ext_discount_amt)] @@ -4529,30 +4445,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manufact_id@1 = 977, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 12), input_partitions=3 @@ -4561,20 +4475,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -4582,25 +4495,26 @@ mod tests { #[tokio::test] async fn test_tpcds_33() -> Result<()> { let display = test_tpcds_query("q33").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [total_sales@1 ASC NULLS LAST], fetch=100 │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(tmp1.total_sales)@1 as total_sales] - │ AggregateExec: mode=SinglePartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(tmp1.total_sales)] - │ InterleaveExec - │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(tmp1.total_sales)@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=SinglePartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 6), input_partitions=3 @@ -4610,7 +4524,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_ext_sales_price@3, i_manufact_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_item_sk@1, ss_ext_sales_price@3] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -4618,48 +4532,45 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 6), input_partitions=3 @@ -4669,7 +4580,7 @@ mod tests { │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_ext_sales_price@3, i_manufact_id@1] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, cs_bill_addr_sk@0)], projection=[cs_item_sk@2, cs_ext_sales_price@3] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -4677,48 +4588,45 @@ mod tests { │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 6), input_partitions=3 @@ -4728,7 +4636,7 @@ mod tests { │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@0)], projection=[ws_ext_sales_price@3, i_manufact_id@1] │ CoalescePartitionsExec - │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ws_bill_addr_sk@1)], projection=[ws_item_sk@1, ws_ext_sales_price@3] │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -4736,48 +4644,45 @@ mod tests { │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 12 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] └────────────────────────────────────────────────── "); Ok(()) @@ -4785,7 +4690,7 @@ mod tests { #[tokio::test] async fn test_tpcds_34() -> Result<()> { let display = test_tpcds_query("q34").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC] │ [Stage 6] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -4794,19 +4699,19 @@ mod tests { │ SortExec: expr=[c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC], preserve_partitioning=[true] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_last_name@3, c_first_name@2, c_salutation@1, c_preferred_cust_flag@4, ss_ticket_number@5, cnt@7] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] │ FilterExec: count(Int64(1))@2 >= 15 AND count(Int64(1))@2 <= 20 │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 12), input_partitions=3 @@ -4821,10 +4726,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -4849,12 +4754,11 @@ mod tests { ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (d_dom@2 >= 1 AND d_dom@2 <= 3 OR d_dom@2 >= 25 AND d_dom@2 <= 28) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -4865,124 +4769,117 @@ mod tests { assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, count(Int64(1))@6 as cnt1, min(customer_demographics.cd_dep_count)@7 as min1, max(customer_demographics.cd_dep_count)@8 as max1, avg(customer_demographics.cd_dep_count)@9 as avg1, cd_dep_employed_count@4 as cd_dep_employed_count, count(Int64(1))@6 as cnt2, min(customer_demographics.cd_dep_employed_count)@10 as min2, max(customer_demographics.cd_dep_employed_count)@11 as max2, avg(customer_demographics.cd_dep_employed_count)@12 as avg2, cd_dep_college_count@5 as cd_dep_college_count, count(Int64(1))@6 as cnt3, min(customer_demographics.cd_dep_college_count)@13 as min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count)@14 as max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)@15 as avg(customer_demographics.cd_dep_college_count)] + │ ProjectionExec: expr=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, count(Int64(1))@6 as cnt1, min(customer_demographics.cd_dep_count)@7 as min1, max(customer_demographics.cd_dep_count)@8 as max1, avg(customer_demographics.cd_dep_count)@9 as avg1, cd_dep_employed_count@4 as cd_dep_employed_count, count(Int64(1))@6 as cnt2, min(customer_demographics.cd_dep_employed_count)@10 as min2, max(customer_demographics.cd_dep_employed_count)@11 as max2, avg(customer_demographics.cd_dep_employed_count)@12 as avg2, cd_dep_college_count@5 as cd_dep_college_count, count(Int64(1))@6 as cnt3, min(customer_demographics.cd_dep_college_count)@13 as min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count)@14 as max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)@15 as avg(customer_demographics.cd_dep_college_count)] + │ SortExec: TopK(fetch=100), expr=[ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@4 ASC, cd_dep_college_count@5 ASC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=3 + ┌───── Stage 11 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] │ FilterExec: mark@6 OR mark@7, projection=[ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5] │ HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(c_customer_sk@0, cs_ship_customer_sk@0)], projection=[ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6, mark@7, mark@8] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=12 + ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftMark, on=[(c_customer_sk@0, ws_bill_customer_sk@0)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, c_customer_sk@0)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@1] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, ca_state@2, cd_gender@4, cd_marital_status@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@2, c_current_cdemo_sk@3, ca_state@1] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@2] + ┌───── Stage 5 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, ca_state@2, cd_gender@4, cd_marital_status@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@2, c_current_cdemo_sk@3, ca_state@1] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 + ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@2] + ┌───── Stage 7 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 + ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=12 + ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_ship_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=48 + ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── "); Ok(()) @@ -5033,15 +4930,15 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6, i_class@1, i_category@2] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -5053,23 +4950,22 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 @@ -5086,15 +4982,15 @@ mod tests { │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6, i_class@1, i_category@2] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -5106,23 +5002,22 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=2, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] @@ -5138,15 +5033,15 @@ mod tests { │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6, i_class@1, i_category@2] │ CoalescePartitionsExec - │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -5158,23 +5053,22 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 12 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── "#); Ok(()) @@ -5182,31 +5076,26 @@ mod tests { #[tokio::test] async fn test_tpcds_37() -> Result<()> { let display = test_tpcds_query("q37").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=2, partitions=3 + ┌───── Stage 6 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=6 + ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@4)], projection=[i_item_sk@1, i_item_id@2, i_item_desc@3, i_current_price@4] + ┌───── Stage 3 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, inv_date_sk@4)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3, inv_date_sk@4] @@ -5214,38 +5103,44 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@3 >= Some(6800),7,2 AND i_current_price@3 <= Some(9800),7,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6800),7,2 AND i_current_price@5 <= Some(9800),7,2 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6800),7,2 AND i_current_price@5 <= Some(9800),7,2 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6800),7,2 AND i_current_price@5 <= Some(9800),7,2 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6800),7,2 AND i_current_price@5 <= Some(9800),7,2 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ FilterExec: i_current_price@3 >= 68.00 AND i_current_price@3 <= 98.00 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 68.00 AND i_current_price@5 <= 98.00 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 68.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 98.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 68.00 AND i_current_price@5 <= 98.00 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 68.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 98.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 68.00 AND i_current_price@5 <= 98.00 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 68.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 98.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 68.00 AND i_current_price@5 <= 98.00 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 68.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 98.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_38() -> Result<()> { let display = test_tpcds_query("q38").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ GlobalLimitExec: skip=0, fetch=100 @@ -5256,13 +5151,11 @@ mod tests { ┌───── Stage 10 ── tasks=2, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], projection=[], NullsEqual: true - │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true - │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -5271,99 +5164,96 @@ mod tests { │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -5371,11 +5261,11 @@ mod tests { #[tokio::test] async fn test_tpcds_39() -> Result<()> { let display = test_tpcds_query("q39").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [wsk1@0 ASC, isk1@1 ASC, dmoy1@2 ASC, mean1@3 ASC, cov1@4 ASC, d_moy@7 ASC, mean@8 ASC, cov@9 ASC] - │ SortExec: expr=[wsk1@0 ASC, isk1@1 ASC, mean1@3 ASC, cov1@4 ASC, mean@8 ASC, cov@9 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[w_warehouse_sk@0 as wsk1, i_item_sk@1 as isk1, d_moy@2 as dmoy1, mean@3 as mean1, cov@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov] + │ ProjectionExec: expr=[w_warehouse_sk@0 as wsk1, i_item_sk@1 as isk1, d_moy@2 as dmoy1, mean@3 as mean1, cov@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov] + │ SortExec: expr=[w_warehouse_sk@0 ASC, i_item_sk@1 ASC, mean@3 ASC, cov@4 ASC, mean@8 ASC, cov@9 ASC], preserve_partitioning=[true] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@1, i_item_sk@1), (w_warehouse_sk@0, w_warehouse_sk@0)] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -5400,22 +5290,21 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4, i_item_sk@0] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 1, projection=[d_date_sk@0, d_moy@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -5425,13 +5314,13 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet:..]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet:..]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 @@ -5453,22 +5342,21 @@ mod tests { │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4, i_item_sk@0] │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0, d_moy@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -5478,13 +5366,13 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet:..]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet:..]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk], file_type=parquet └────────────────────────────────────────────────── "); Ok(()) @@ -5498,14 +5386,15 @@ mod tests { │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[w_state@0 as w_state, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@2 as sales_before, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@3 as sales_after] - │ AggregateExec: mode=FinalPartitioned, gby=[w_state@0 as w_state, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN __common_expr_1 < 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE Some(0),22,2 END ELSE Some(0),23,2 END) as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 >= 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE Some(0),22,2 END ELSE Some(0),23,2 END) as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[w_state@0 as w_state, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN __common_expr_1 < 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE 0.00 END ELSE 0.00 END) as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 >= 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE 0.00 END ELSE 0.00 END) as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([w_state@0, i_item_id@1], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[w_state@3 as w_state, i_item_id@4 as i_item_id], aggr=[sum(CASE WHEN __common_expr_1 < 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE Some(0),22,2 END ELSE Some(0),23,2 END) as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 >= 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE Some(0),22,2 END ELSE Some(0),23,2 END) as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ AggregateExec: mode=Partial, gby=[w_state@3 as w_state, i_item_id@4 as i_item_id], aggr=[sum(CASE WHEN __common_expr_1 < 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE 0.00 END ELSE 0.00 END) as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 >= 2000-03-11 THEN catalog_sales.cs_sales_price - CASE WHEN catalog_returns.cr_refunded_cash IS NOT NULL THEN catalog_returns.cr_refunded_cash ELSE 0.00 END ELSE 0.00 END) as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] │ ProjectionExec: expr=[d_date@0 as __common_expr_1, cs_sales_price@1 as cs_sales_price, cr_refunded_cash@2 as cr_refunded_cash, w_state@3 as w_state, i_item_id@4 as i_item_id] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[d_date@1, cs_sales_price@3, cr_refunded_cash@4, w_state@5, i_item_id@6] │ CoalescePartitionsExec @@ -5523,22 +5412,20 @@ mod tests { ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@2 >= Some(99),7,2 AND i_current_price@2 <= Some(149),7,2, projection=[i_item_sk@0, i_item_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),7,2 AND i_current_price@5 <= Some(149),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),7,2, required_guarantees=[] + │ FilterExec: i_current_price@2 >= 0.99 AND i_current_price@2 <= 1.49, projection=[i_item_sk@0, i_item_id@1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= 0.99 AND i_current_price@5 <= 1.49, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 0.99 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 1.49, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -5559,10 +5446,10 @@ mod tests { ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "#); Ok(()) @@ -5585,12 +5472,11 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=1 │ FilterExec: i_manufact_id@0 >= 738 AND i_manufact_id@0 <= 778, projection=[i_manufact@1, i_product_name@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=1, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -5602,13 +5488,11 @@ mod tests { │ RepartitionExec: partitioning=Hash([i_manufact@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_manufact@0 as i_manufact], aggr=[count(Int64(1))] │ FilterExec: __common_expr_4@0 AND ((i_color@4 = powder OR i_color@4 = khaki) AND (i_units@5 = Ounce OR i_units@5 = Oz) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = brown OR i_color@4 = honeydew) AND (i_units@5 = Bunch OR i_units@5 = Ton) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = floral OR i_color@4 = deep) AND (i_units@5 = N/A OR i_units@5 = Dozen) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = light OR i_color@4 = cornflower) AND (i_units@5 = Box OR i_units@5 = Pound) AND (i_size@3 = medium OR i_size@3 = extra large) OR __common_expr_4@0 AND ((i_color@4 = midnight OR i_color@4 = snow) AND (i_units@5 = Pallet OR i_units@5 = Gross) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = cyan OR i_color@4 = papaya) AND (i_units@5 = Cup OR i_units@5 = Dram) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = orange OR i_color@4 = frosted) AND (i_units@5 = Each OR i_units@5 = Tbl) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = forest OR i_color@4 = ghost) AND (i_units@5 = Lb OR i_units@5 = Bundle) AND (i_size@3 = medium OR i_size@3 = extra large), projection=[i_manufact@2] - │ ProjectionExec: expr=[i_category@0 = Women as __common_expr_4, i_category@0 as i_category, i_manufact@1 as i_manufact, i_size@2 as i_size, i_color@3 as i_color, i_units@4 as i_units] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category@12 = Women as __common_expr_4, i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_category@12 = Women as __common_expr_4, i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_category@12 = Women as __common_expr_4, i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_category@12 = Women as __common_expr_4, i_category, i_manufact, i_size, i_color, i_units], file_type=parquet, predicate=i_category@12 = Women AND ((i_color@17 = powder OR i_color@17 = khaki) AND (i_units@18 = Ounce OR i_units@18 = Oz) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = brown OR i_color@17 = honeydew) AND (i_units@18 = Bunch OR i_units@18 = Ton) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = floral OR i_color@17 = deep) AND (i_units@18 = N/A OR i_units@18 = Dozen) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = light OR i_color@17 = cornflower) AND (i_units@18 = Box OR i_units@18 = Pound) AND (i_size@15 = medium OR i_size@15 = extra large) OR i_category@12 = Women AND ((i_color@17 = midnight OR i_color@17 = snow) AND (i_units@18 = Pallet OR i_units@18 = Gross) AND (i_size@15 = medium OR i_size@15 = extra large) OR (i_color@17 = cyan OR i_color@17 = papaya) AND (i_units@18 = Cup OR i_units@18 = Dram) AND (i_size@15 = N/A OR i_size@15 = small)) OR i_category@12 = Men AND (i_color@17 = orange OR i_color@17 = frosted) AND (i_units@18 = Each OR i_units@18 = Tbl) AND i_size@15 = petite OR i_category@12 = Men AND (i_color@17 = forest OR i_color@17 = ghost) AND (i_units@18 = Lb OR i_units@18 = Bundle) AND (i_size@15 = medium OR i_size@15 = extra large), pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= powder AND powder <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= khaki AND khaki <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ounce AND Ounce <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Oz AND Oz <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= brown AND brown <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= honeydew AND honeydew <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bunch AND Bunch <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Ton AND Ton <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= floral AND floral <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= deep AND deep <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= N/A AND N/A <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dozen AND Dozen <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= light AND light <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cornflower AND cornflower <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Box AND Box <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pound AND Pound <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND ((i_color_null_count@6 != row_count@3 AND i_color_min@4 <= midnight AND midnight <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= snow AND snow <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Pallet AND Pallet <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Gross AND Gross <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11) OR (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= cyan AND cyan <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= papaya AND papaya <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Cup AND Cup <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Dram AND Dram <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= N/A AND N/A <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= small AND small <= i_size_max@11)) OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= orange AND orange <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= frosted AND frosted <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Each AND Each <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Tbl AND Tbl <= i_units_max@8) AND i_size_null_count@12 != row_count@3 AND i_size_min@10 <= petite AND petite <= i_size_max@11 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 AND (i_color_null_count@6 != row_count@3 AND i_color_min@4 <= forest AND forest <= i_color_max@5 OR i_color_null_count@6 != row_count@3 AND i_color_min@4 <= ghost AND ghost <= i_color_max@5) AND (i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Lb AND Lb <= i_units_max@8 OR i_units_null_count@9 != row_count@3 AND i_units_min@7 <= Bundle AND Bundle <= i_units_max@8) AND (i_size_null_count@12 != row_count@3 AND i_size_min@10 <= medium AND medium <= i_size_max@11 OR i_size_null_count@12 != row_count@3 AND i_size_min@10 <= extra large AND extra large <= i_size_max@11), required_guarantees=[i_category in (Men, Women)] └────────────────────────────────────────────────── "); Ok(()) @@ -5636,30 +5520,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_category_id@1, i_category@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] └────────────────────────────────────────────────── "); Ok(()) @@ -5673,14 +5555,15 @@ mod tests { │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([s_store_name@0, s_store_id@1], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[s_store_name@3 as s_store_name, s_store_id@2 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ AggregateExec: mode=Partial, gby=[s_store_name@3 as s_store_name, s_store_id@2 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[d_day_name@3, ss_sales_price@5, s_store_id@1, s_store_name@2] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -5688,30 +5571,29 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: s_gmt_offset@3 = Some(-500),5,2, projection=[s_store_sk@0, s_store_id@1, s_store_name@2] + │ FilterExec: s_gmt_offset@3 = -5.00, projection=[s_store_sk@0, s_store_id@1, s_store_name@2] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2 AND DynamicFilter [ empty ], pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2 AND DynamicFilter [ empty ], pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2 AND DynamicFilter [ empty ], pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2 AND DynamicFilter [ empty ], pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0, d_day_name@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── "#); Ok(()) @@ -5729,29 +5611,29 @@ mod tests { │ ProjectionExec: expr=[rnk@0 as rnk, i_product_name@1 as best_performing, i_product_name@2 as worst_performing] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@1)], projection=[rnk@2, i_product_name@4, i_product_name@1] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@0)], projection=[rnk@3, item_sk@4, i_product_name@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(rnk@1, rnk@1)], projection=[item_sk@0, rnk@1, item_sk@2] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=1 │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=1 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_product_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=1, partitions=12 │ RepartitionExec: partitioning=Hash([rnk@1], 12), input_partitions=3 @@ -5762,14 +5644,16 @@ mod tests { │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortPreservingMergeExec: [rank_col@1 ASC NULLS LAST] │ SortExec: expr=[rank_col@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] - │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] - │ NestedLoopJoinExec: join_type=Right, projection=[ss_item_sk@1, avg(ss1.ss_net_profit)@2, rank_col@0] - │ CoalescePartitionsExec - │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 - │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] - │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[ss_item_sk@1 as item_sk, avg(ss1.ss_net_profit)@2 as rank_col] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_2@1 > join_proj_push_down_1@0, projection=[rank_col@0, ss_item_sk@2, avg(ss1.ss_net_profit)@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 + │ ProjectionExec: expr=[rank_col@0 as rank_col, CAST(0.9 * rank_col@0 AS Decimal128(30, 15)) as join_proj_push_down_1] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[ss_item_sk@0 as ss_item_sk, avg(ss1.ss_net_profit)@1 as avg(ss1.ss_net_profit), CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) as join_proj_push_down_2] │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -5802,14 +5686,16 @@ mod tests { │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortPreservingMergeExec: [rank_col@1 DESC] │ SortExec: expr=[rank_col@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] - │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] - │ NestedLoopJoinExec: join_type=Right, projection=[ss_item_sk@1, avg(ss1.ss_net_profit)@2, rank_col@0] - │ CoalescePartitionsExec - │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 - │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] - │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[ss_item_sk@1 as item_sk, avg(ss1.ss_net_profit)@2 as rank_col] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_4@1 > join_proj_push_down_3@0, projection=[rank_col@0, ss_item_sk@2, avg(ss1.ss_net_profit)@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 + │ ProjectionExec: expr=[rank_col@0 as rank_col, CAST(0.9 * rank_col@0 AS Decimal128(30, 15)) as join_proj_push_down_3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[ss_item_sk@0 as ss_item_sk, avg(ss1.ss_net_profit)@1 as avg(ss1.ss_net_profit), CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) as join_proj_push_down_4] │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -5839,7 +5725,7 @@ mod tests { #[tokio::test] async fn test_tpcds_45() -> Result<()> { let display = test_tpcds_query("q45").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], fetch=100 │ SortExec: TopK(fetch=100), expr=[ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], preserve_partitioning=[true] @@ -5855,65 +5741,63 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@0)], projection=[ws_sales_price@3, ca_city@4, ca_zip@5, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_sales_price@3, ca_city@4, ca_zip@5] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@5, ca_city@1, ca_zip@2] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@2)], projection=[ws_sold_date_sk@2, ws_item_sk@3, ws_sales_price@5, c_current_addr_sk@1] │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), projection=[i_item_id@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 5 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -5921,7 +5805,7 @@ mod tests { #[tokio::test] async fn test_tpcds_46() -> Result<()> { let display = test_tpcds_query("q46").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], fetch=100 │ [Stage 8] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -5930,36 +5814,36 @@ mod tests { │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@4)], filter=bought_city@0 != ca_city@1, projection=[c_last_name@8, c_first_name@7, ca_city@1, bought_city@3, ss_ticket_number@2, amt@4, profit@5] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_ticket_number@4, bought_city@6, amt@7, profit@8, c_current_addr_sk@1, c_first_name@2, c_last_name@3] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 12), input_partitions=3 │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@5 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@3, ss_ticket_number@4, ss_coupon_amt@5, ss_net_profit@6, ca_city@1] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, ss_hdemo_sk@1)], projection=[ss_customer_sk@1, ss_addr_sk@3, ss_ticket_number@4, ss_coupon_amt@5, ss_net_profit@6] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -5970,18 +5854,18 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -6006,12 +5890,11 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (d_dow@2 = 6 OR d_dow@2 = 0) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -6024,26 +5907,27 @@ mod tests { │ SortPreservingMergeExec: [sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], fetch=100 │ SortExec: TopK(fetch=100), expr=[sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, avg_monthly_sales@6 as avg_monthly_sales, sum_sales@7 as sum_sales, sum_sales@8 as psum, sum_sales@9 as nsum] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@10, v1_lead.rn - Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, avg_monthly_sales@7, sum_sales@6, sum_sales@9, sum_sales@15] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@10, v1_lead.rn - Decimal128(1,20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, avg_monthly_sales@7, sum_sales@6, sum_sales@9, sum_sales@15] │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, CAST(v1.rn AS Decimal128(21, 0))@10], 3), input_partitions=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum_sales@6 as sum_sales, avg_monthly_sales@7 as avg_monthly_sales, rn@8 as rn, sum_sales@9 as sum_sales, CAST(rn@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lag.rn + Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, rn@8, sum_sales@14] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lag.rn + Decimal128(1,20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, rn@8, sum_sales@14] │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, CAST(v1.rn AS Decimal128(21, 0))@9], 3), input_partitions=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] - │ FilterExec: d_year@4 = 1999 AND avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),21,6 AND CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),21,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 END > Some(1000000000),32,10 - │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 - │ BoundedWindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(21, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, v1_lag.rn + Decimal128(Some(1),20,0)@6], 3), input_partitions=3 - │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ FilterExec: __common_expr_3@0 AND d_year@5 = 1999 AND CASE WHEN __common_expr_3@0 THEN abs(sum(store_sales.ss_sales_price)@7 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 END > 0.1000000000, projection=[i_category@1, i_brand@2, s_store_name@3, s_company_name@4, d_year@5, d_moy@6, sum(store_sales.ss_sales_price)@7, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9] + │ ProjectionExec: expr=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > 0.000000 as __common_expr_3, i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum(store_sales.ss_sales_price), avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 as rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(21, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, v1_lag.rn + Decimal128(1,20,0)@6], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) + 1 as v1_lag.rn + Decimal128(1,20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, v1_lead.rn - Decimal128(Some(1),20,0)@6], 3), input_partitions=3 - │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, v1_lead.rn - Decimal128(1,20,0)@6], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) - 1 as v1_lead.rn - Decimal128(1,20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -6064,12 +5948,12 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6082,20 +5966,19 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 @@ -6113,12 +5996,12 @@ mod tests { │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6131,20 +6014,19 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 @@ -6162,12 +6044,12 @@ mod tests { │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6180,20 +6062,19 @@ mod tests { ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── "#); Ok(()) @@ -6212,32 +6093,30 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_quantity@2] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], filter=(ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX) AND ss_net_profit@0 >= Some(0),7,2 AND ss_net_profit@0 <= Some(200000),7,2 OR (ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),7,2 AND ss_net_profit@0 <= Some(300000),7,2 OR (ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),7,2 AND ss_net_profit@0 <= Some(2500000),7,2, projection=[ss_sold_date_sk@2, ss_quantity@4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], filter=(ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX) AND ss_net_profit@0 >= 0.00 AND ss_net_profit@0 <= 2000.00 OR (ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY) AND ss_net_profit@0 >= 150.00 AND ss_net_profit@0 <= 3000.00 OR (ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ss_net_profit@0 >= 50.00 AND ss_net_profit@0 <= 25000.00, projection=[ss_sold_date_sk@2, ss_quantity@4] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree AND ss_sales_price@0 >= Some(10000),7,2 AND ss_sales_price@0 <= Some(15000),7,2 OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(5000),7,2 AND ss_sales_price@0 <= Some(10000),7,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(15000),7,2 AND ss_sales_price@0 <= Some(20000),7,2, projection=[ss_sold_date_sk@3, ss_addr_sk@5, ss_quantity@6, ss_net_profit@8] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], filter=cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree AND ss_sales_price@0 >= 100.00 AND ss_sales_price@0 <= 150.00 OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= 50.00 AND ss_sales_price@0 <= 100.00 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= 150.00 AND ss_sales_price@0 <= 200.00, projection=[ss_sold_date_sk@3, ss_addr_sk@5, ss_quantity@6, ss_net_profit@8] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_country@2 = United States AND ca_state@1 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX OR ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY OR ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS), projection=[ca_address_sk@0, ca_state@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND (ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CO AND CO <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MN AND MN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CA AND CA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 @@ -6246,19 +6125,19 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@3)], projection=[ss_sold_date_sk@1, ss_cdemo_sk@2, ss_addr_sk@3, ss_quantity@5, ss_sales_price@6, ss_net_profit@7] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ FilterExec: (ss_net_profit@6 >= Some(0),7,2 AND ss_net_profit@6 <= Some(200000),7,2 OR ss_net_profit@6 >= Some(15000),7,2 AND ss_net_profit@6 <= Some(300000),7,2 OR ss_net_profit@6 >= Some(5000),7,2 AND ss_net_profit@6 <= Some(2500000),7,2) AND (ss_sales_price@5 >= Some(10000),7,2 AND ss_sales_price@5 <= Some(15000),7,2 OR ss_sales_price@5 >= Some(5000),7,2 AND ss_sales_price@5 <= Some(10000),7,2 OR ss_sales_price@5 >= Some(15000),7,2 AND ss_sales_price@5 <= Some(20000),7,2) + │ FilterExec: (ss_net_profit@6 >= 0.00 AND ss_net_profit@6 <= 2000.00 OR ss_net_profit@6 >= 150.00 AND ss_net_profit@6 <= 3000.00 OR ss_net_profit@6 >= 50.00 AND ss_net_profit@6 <= 25000.00) AND (ss_sales_price@5 >= 100.00 AND ss_sales_price@5 <= 150.00 OR ss_sales_price@5 >= 50.00 AND ss_sales_price@5 <= 100.00 OR ss_sales_price@5 >= 150.00 AND ss_sales_price@5 <= 200.00) │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(0),7,2 AND ss_net_profit@22 <= Some(200000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(300000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(2500000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(2500000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(0),7,2 AND ss_net_profit@22 <= Some(200000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(300000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(2500000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(2500000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(0),7,2 AND ss_net_profit@22 <= Some(200000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(300000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(2500000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(2500000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(0),7,2 AND ss_net_profit@22 <= Some(200000),7,2 OR ss_net_profit@22 >= Some(15000),7,2 AND ss_net_profit@22 <= Some(300000),7,2 OR ss_net_profit@22 >= Some(5000),7,2 AND ss_net_profit@22 <= Some(2500000),7,2) AND (ss_sales_price@13 >= Some(10000),7,2 AND ss_sales_price@13 <= Some(15000),7,2 OR ss_sales_price@13 >= Some(5000),7,2 AND ss_sales_price@13 <= Some(10000),7,2 OR ss_sales_price@13 >= Some(15000),7,2 AND ss_sales_price@13 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),7,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),7,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(2500000),7,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),7,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),7,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 0.00 AND ss_net_profit@22 <= 2000.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 3000.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 25000.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 0.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 2000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 3000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 25000.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 0.00 AND ss_net_profit@22 <= 2000.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 3000.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 25000.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 0.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 2000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 3000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 25000.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 0.00 AND ss_net_profit@22 <= 2000.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 3000.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 25000.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 0.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 2000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 3000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 25000.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= 0.00 AND ss_net_profit@22 <= 2000.00 OR ss_net_profit@22 >= 150.00 AND ss_net_profit@22 <= 3000.00 OR ss_net_profit@22 >= 50.00 AND ss_net_profit@22 <= 25000.00) AND (ss_sales_price@13 >= 100.00 AND ss_sales_price@13 <= 150.00 OR ss_sales_price@13 >= 50.00 AND ss_sales_price@13 <= 100.00 OR ss_sales_price@13 >= 150.00 AND ss_sales_price@13 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 0.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 2000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 150.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 3000.00 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= 50.00 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= 25000.00) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 100.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 150.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 50.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 100.00 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= 150.00 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= 200.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6279,9 +6158,9 @@ mod tests { │ SortPreservingMergeExec: [channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[] - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=3, partitions=3 + ┌───── Stage 13 ── tasks=3, partitions=3 │ RepartitionExec: partitioning=Hash([channel@0, item@1, return_ratio@2, return_rank@3, currency_rank@4], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[], ordering_mode=PartiallySorted([0, 4]) │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] @@ -6293,7 +6172,7 @@ mod tests { │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] │ FilterExec: rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true @@ -6302,7 +6181,7 @@ mod tests { │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] │ FilterExec: rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true @@ -6311,159 +6190,148 @@ mod tests { │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(CASE WHEN __common_expr_9 IS NOT NULL THEN __common_expr_9 ELSE 0 END) as sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_10 IS NOT NULL THEN __common_expr_10 ELSE 0 END) as sum(coalesce(sts.ss_quantity,Int64(0))), sum(CASE WHEN __common_expr_11 IS NOT NULL THEN __common_expr_11 ELSE 0.00 END) as sum(coalesce(sr.sr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_12 IS NOT NULL THEN __common_expr_12 ELSE 0.00 END) as sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=2, partitions=3 + ┌───── Stage 4 ── tasks=2, partitions=3 │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[ws_item_sk@0 as item, CAST(sum(coalesce(wr.wr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(wr.wr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0 END) as sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0 END) as sum(coalesce(ws.ws_quantity,Int64(0))), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Some(0),22,2 END) as sum(coalesce(wr.wr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE Some(0),22,2 END) as sum(coalesce(ws.ws_net_paid,Int64(0)))] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0 END) as sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0 END) as sum(coalesce(ws.ws_quantity,Int64(0))), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE 0.00 END) as sum(coalesce(wr.wr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE 0.00 END) as sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=6 + ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ws_item_sk@4 as ws_item_sk], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0 END) as sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0 END) as sum(coalesce(ws.ws_quantity,Int64(0))), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Some(0),22,2 END) as sum(coalesce(wr.wr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE Some(0),22,2 END) as sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[ws_item_sk@4 as ws_item_sk], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0 END) as sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0 END) as sum(coalesce(ws.ws_quantity,Int64(0))), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE 0.00 END) as sum(coalesce(wr.wr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE 0.00 END) as sum(coalesce(ws.ws_net_paid,Int64(0)))] │ ProjectionExec: expr=[CAST(wr_return_quantity@0 AS Int64) as __common_expr_1, CAST(ws_quantity@1 AS Int64) as __common_expr_2, CAST(wr_return_amt@2 AS Decimal128(22, 2)) as __common_expr_3, CAST(ws_net_paid@3 AS Decimal128(22, 2)) as __common_expr_4, ws_item_sk@4 as ws_item_sk] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[wr_return_quantity@5, ws_quantity@3, wr_return_amt@6, ws_net_paid@4, ws_item_sk@2] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_order_number@1, ws_order_number@2), (wr_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@7, ws_net_paid@8, wr_return_quantity@2, wr_return_amt@3] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_order_number@1, ws_order_number@2), (wr_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@7, ws_net_paid@8, wr_return_quantity@2, wr_return_amt@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ FilterExec: ws_net_profit@5 > 1.00 AND ws_net_paid@4 > 0.00 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > 1.00 AND ws_net_paid@29 > 0.00 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > 1.00 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > 0.00 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > 1.00 AND ws_net_paid@29 > 0.00 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > 1.00 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > 0.00 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > 1.00 AND ws_net_paid@29 > 0.00 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > 1.00 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > 0.00 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > 1.00 AND ws_net_paid@29 > 0.00 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > 1.00 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > 0.00 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 12), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 12), input_partitions=3 - │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: wr_return_amt@3 > 10000.00 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > Some(100),7,2 AND ws_net_paid@29 > Some(0),7,2 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > Some(100),7,2 AND ws_net_paid@29 > Some(0),7,2 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > Some(100),7,2 AND ws_net_paid@29 > Some(0),7,2 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > Some(100),7,2 AND ws_net_paid@29 > Some(0),7,2 AND ws_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=wr_return_amt@15 > 10000.00, pruning_predicate=wr_return_amt_null_count@1 != row_count@2 AND wr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=wr_return_amt@15 > 10000.00, pruning_predicate=wr_return_amt_null_count@1 != row_count@2 AND wr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=wr_return_amt@15 > 10000.00, pruning_predicate=wr_return_amt_null_count@1 != row_count@2 AND wr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=wr_return_amt@15 > 10000.00, pruning_predicate=wr_return_amt_null_count@1 != row_count@2 AND wr_return_amt_max@0 > 10000.00, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=2, partitions=3 + ┌───── Stage 8 ── tasks=2, partitions=3 │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[cs_item_sk@0 as item, CAST(sum(coalesce(cr.cr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(cr.cr_return_amount,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0 END) as sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0 END) as sum(coalesce(cs.cs_quantity,Int64(0))), sum(CASE WHEN __common_expr_7 IS NOT NULL THEN __common_expr_7 ELSE Some(0),22,2 END) as sum(coalesce(cr.cr_return_amount,Int64(0))), sum(CASE WHEN __common_expr_8 IS NOT NULL THEN __common_expr_8 ELSE Some(0),22,2 END) as sum(coalesce(cs.cs_net_paid,Int64(0)))] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0 END) as sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0 END) as sum(coalesce(cs.cs_quantity,Int64(0))), sum(CASE WHEN __common_expr_7 IS NOT NULL THEN __common_expr_7 ELSE 0.00 END) as sum(coalesce(cr.cr_return_amount,Int64(0))), sum(CASE WHEN __common_expr_8 IS NOT NULL THEN __common_expr_8 ELSE 0.00 END) as sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=6 + ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cs_item_sk@4 as cs_item_sk], aggr=[sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0 END) as sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0 END) as sum(coalesce(cs.cs_quantity,Int64(0))), sum(CASE WHEN __common_expr_7 IS NOT NULL THEN __common_expr_7 ELSE Some(0),22,2 END) as sum(coalesce(cr.cr_return_amount,Int64(0))), sum(CASE WHEN __common_expr_8 IS NOT NULL THEN __common_expr_8 ELSE Some(0),22,2 END) as sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[cs_item_sk@4 as cs_item_sk], aggr=[sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0 END) as sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0 END) as sum(coalesce(cs.cs_quantity,Int64(0))), sum(CASE WHEN __common_expr_7 IS NOT NULL THEN __common_expr_7 ELSE 0.00 END) as sum(coalesce(cr.cr_return_amount,Int64(0))), sum(CASE WHEN __common_expr_8 IS NOT NULL THEN __common_expr_8 ELSE 0.00 END) as sum(coalesce(cs.cs_net_paid,Int64(0)))] │ ProjectionExec: expr=[CAST(cr_return_quantity@0 AS Int64) as __common_expr_5, CAST(cs_quantity@1 AS Int64) as __common_expr_6, CAST(cr_return_amount@2 AS Decimal128(22, 2)) as __common_expr_7, CAST(cs_net_paid@3 AS Decimal128(22, 2)) as __common_expr_8, cs_item_sk@4 as cs_item_sk] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cr_return_quantity@5, cs_quantity@3, cr_return_amount@6, cs_net_paid@4, cs_item_sk@2] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@2), (cr_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@7, cs_net_paid@8, cr_return_quantity@2, cr_return_amount@3] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_order_number@1, cs_order_number@2), (cr_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@7, cs_net_paid@8, cr_return_quantity@2, cr_return_amount@3] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ FilterExec: cs_net_profit@5 > 1.00 AND cs_net_paid@4 > 0.00 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > 1.00 AND cs_net_paid@29 > 0.00 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > 1.00 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > 0.00 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > 1.00 AND cs_net_paid@29 > 0.00 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > 1.00 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > 0.00 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > 1.00 AND cs_net_paid@29 > 0.00 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > 1.00 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > 0.00 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > 1.00 AND cs_net_paid@29 > 0.00 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > 1.00 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > 0.00 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=48 + ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 12), input_partitions=3 - │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > Some(100),7,2 AND cs_net_paid@29 > Some(0),7,2 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > Some(100),7,2 AND cs_net_paid@29 > Some(0),7,2 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > Some(100),7,2 AND cs_net_paid@29 > Some(0),7,2 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > Some(100),7,2 AND cs_net_paid@29 > Some(0),7,2 AND cs_quantity@18 > 0 AND DynamicFilter [ empty ], pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=2, partitions=3 - │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(CASE WHEN __common_expr_9 IS NOT NULL THEN __common_expr_9 ELSE 0 END) as sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_10 IS NOT NULL THEN __common_expr_10 ELSE 0 END) as sum(coalesce(sts.ss_quantity,Int64(0))), sum(CASE WHEN __common_expr_11 IS NOT NULL THEN __common_expr_11 ELSE Some(0),22,2 END) as sum(coalesce(sr.sr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_12 IS NOT NULL THEN __common_expr_12 ELSE Some(0),22,2 END) as sum(coalesce(sts.ss_net_paid,Int64(0)))] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ss_item_sk@4 as ss_item_sk], aggr=[sum(CASE WHEN __common_expr_9 IS NOT NULL THEN __common_expr_9 ELSE 0 END) as sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_10 IS NOT NULL THEN __common_expr_10 ELSE 0 END) as sum(coalesce(sts.ss_quantity,Int64(0))), sum(CASE WHEN __common_expr_11 IS NOT NULL THEN __common_expr_11 ELSE Some(0),22,2 END) as sum(coalesce(sr.sr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_12 IS NOT NULL THEN __common_expr_12 ELSE Some(0),22,2 END) as sum(coalesce(sts.ss_net_paid,Int64(0)))] - │ ProjectionExec: expr=[CAST(sr_return_quantity@0 AS Int64) as __common_expr_9, CAST(ss_quantity@1 AS Int64) as __common_expr_10, CAST(sr_return_amt@2 AS Decimal128(22, 2)) as __common_expr_11, CAST(ss_net_paid@3 AS Decimal128(22, 2)) as __common_expr_12, ss_item_sk@4 as ss_item_sk] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[sr_return_quantity@5, ss_quantity@3, sr_return_amt@6, ss_net_paid@4, ss_item_sk@2] - │ CoalescePartitionsExec - │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: sr_return_amt@5 > Some(1000000),7,2 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8, sr_return_quantity@2, sr_return_amt@3] - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=48 + ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 12), input_partitions=3 - │ FilterExec: ss_net_profit@5 > Some(100),7,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] + │ FilterExec: cr_return_amount@3 > 10000.00 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > Some(100),7,2 AND ss_net_paid@20 > Some(0),7,2 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ], pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),7,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > Some(100),7,2 AND ss_net_paid@20 > Some(0),7,2 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ], pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),7,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > Some(100),7,2 AND ss_net_paid@20 > Some(0),7,2 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ], pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),7,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > Some(100),7,2 AND ss_net_paid@20 > Some(0),7,2 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ], pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),7,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] - └────────────────────────────────────────────────── + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=cr_return_amount@18 > 10000.00, pruning_predicate=cr_return_amount_null_count@1 != row_count@2 AND cr_return_amount_max@0 > 10000.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=cr_return_amount@18 > 10000.00, pruning_predicate=cr_return_amount_null_count@1 != row_count@2 AND cr_return_amount_max@0 > 10000.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=cr_return_amount@18 > 10000.00, pruning_predicate=cr_return_amount_null_count@1 != row_count@2 AND cr_return_amount_max@0 > 10000.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=cr_return_amount@18 > 10000.00, pruning_predicate=cr_return_amount_null_count@1 != row_count@2 AND cr_return_amount_max@0 > 10000.00, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── tasks=2, partitions=3 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@4 as ss_item_sk], aggr=[sum(CASE WHEN __common_expr_9 IS NOT NULL THEN __common_expr_9 ELSE 0 END) as sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(CASE WHEN __common_expr_10 IS NOT NULL THEN __common_expr_10 ELSE 0 END) as sum(coalesce(sts.ss_quantity,Int64(0))), sum(CASE WHEN __common_expr_11 IS NOT NULL THEN __common_expr_11 ELSE 0.00 END) as sum(coalesce(sr.sr_return_amt,Int64(0))), sum(CASE WHEN __common_expr_12 IS NOT NULL THEN __common_expr_12 ELSE 0.00 END) as sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ ProjectionExec: expr=[CAST(sr_return_quantity@0 AS Int64) as __common_expr_9, CAST(ss_quantity@1 AS Int64) as __common_expr_10, CAST(sr_return_amt@2 AS Decimal128(22, 2)) as __common_expr_11, CAST(ss_net_paid@3 AS Decimal128(22, 2)) as __common_expr_12, ss_item_sk@4 as ss_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[sr_return_quantity@5, ss_quantity@3, sr_return_amt@6, ss_net_paid@4, ss_item_sk@2] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8, sr_return_quantity@2, sr_return_amt@3] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── tasks=4, partitions=24 + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 6), input_partitions=3 + │ FilterExec: sr_return_amt@3 > 10000.00 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=sr_return_amt@11 > 10000.00, pruning_predicate=sr_return_amt_null_count@1 != row_count@2 AND sr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=sr_return_amt@11 > 10000.00, pruning_predicate=sr_return_amt_null_count@1 != row_count@2 AND sr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=sr_return_amt@11 > 10000.00, pruning_predicate=sr_return_amt_null_count@1 != row_count@2 AND sr_return_amt_max@0 > 10000.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=sr_return_amt@11 > 10000.00, pruning_predicate=sr_return_amt_null_count@1 != row_count@2 AND sr_return_amt_max@0 > 10000.00, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 6), input_partitions=3 + │ FilterExec: ss_net_profit@5 > 1.00 AND ss_net_paid@4 > 0.00 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > 1.00 AND ss_net_paid@20 > 0.00 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > 1.00 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > 0.00 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > 1.00 AND ss_net_paid@20 > 0.00 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > 1.00 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > 0.00 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > 1.00 AND ss_net_paid@20 > 0.00 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > 1.00 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > 0.00 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > 1.00 AND ss_net_paid@20 > 0.00 AND ss_quantity@10 > 0 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > 1.00 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > 0.00 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── "#); Ok(()) } #[tokio::test] async fn test_tpcds_50() -> Result<()> { let display = test_tpcds_query("q50").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], fetch=100 │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@10 as 30 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@11 as 31-60 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@12 as 61-90 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@13 as 91-120 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@14 as >120 days] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip], aggr=[sum(CASE WHEN __common_expr_1 <= 30 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 30 AND __common_expr_1 <= 60 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 60 AND __common_expr_1 <= 90 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 90 AND __common_expr_1 <= 120 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 120 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip], aggr=[sum(CASE WHEN __common_expr_1 <= 30 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 30 AND __common_expr_1 <= 60 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 60 AND __common_expr_1 <= 90 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 90 AND __common_expr_1 <= 120 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 120 THEN 1 ELSE 0 END) as sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([s_store_name@0, s_company_id@1, s_street_number@2, s_street_name@3, s_street_type@4, s_suite_number@5, s_city@6, s_county@7, s_state@8, s_zip@9], 6), input_partitions=3 @@ -6474,7 +6342,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_sold_date_sk@1, sr_returned_date_sk@2, s_store_name@3, s_company_id@4, s_street_number@5, s_street_name@6, s_street_type@7, s_suite_number@8, s_city@9, s_county@10, s_state@11, s_zip@12] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_sold_date_sk@11, sr_returned_date_sk@13, s_store_name@1, s_company_id@2, s_street_number@3, s_street_name@4, s_street_type@5, s_suite_number@6, s_city@7, s_county@8, s_state@9, s_zip@10] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -6485,44 +6353,43 @@ mod tests { ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 8, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_ticket_number@3, sr_item_sk@1, sr_customer_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@4, ss_item_sk@1, ss_customer_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -6533,8 +6400,8 @@ mod tests { assert_snapshot!(display, @r#" ┌───── DistributedExec │ SortPreservingMergeExec: [item_sk@0 ASC, d_date@1 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[item_sk@0 ASC, d_date@1 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[item_sk@0 as item_sk, d_date@1 as d_date, web_sales@2 as web_sales, store_sales@3 as store_sales, max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as web_cumulative, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as store_cumulative] + │ ProjectionExec: expr=[item_sk@0 as item_sk, d_date@1 as d_date, web_sales@2 as web_sales, store_sales@3 as store_sales, max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as web_cumulative, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as store_cumulative] + │ SortExec: TopK(fetch=100), expr=[item_sk@0 ASC, d_date@1 ASC], preserve_partitioning=[true] │ FilterExec: max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 > max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 │ BoundedWindowAggExec: wdw=[max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(27, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(27, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] @@ -6565,20 +6432,19 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: ws_item_sk@1 IS NOT NULL │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 @@ -6593,20 +6459,19 @@ mod tests { │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: ss_item_sk@1 IS NOT NULL │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "#); Ok(()) @@ -6614,16 +6479,17 @@ mod tests { #[tokio::test] async fn test_tpcds_52() -> Result<()> { let display = test_tpcds_query("q52").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, ext_price@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[ext_price@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as ext_price] - │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(store_sales.ss_ext_sales_price)@3 DESC, i_brand_id@2 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 6), input_partitions=3 @@ -6635,30 +6501,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] └────────────────────────────────────────────────── "); Ok(()) @@ -6669,9 +6533,9 @@ mod tests { assert_snapshot!(display, @r#" ┌───── DistributedExec │ SortPreservingMergeExec: [avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], fetch=100 - │ SortExec: TopK(fetch=100), expr=[avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_quarterly_sales] - │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),21,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),32,10 + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_quarterly_sales] + │ SortExec: TopK(fetch=100), expr=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 ASC NULLS LAST, sum(store_sales.ss_sales_price)@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > 0.000000 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > 0.1000000000 │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(21, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] │ SortExec: expr=[i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -6695,10 +6559,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6711,22 +6575,20 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_qoy@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manufact_id@4] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] └────────────────────────────────────────────────── "#); Ok(()) @@ -6734,7 +6596,7 @@ mod tests { #[tokio::test] async fn test_tpcds_54() -> Result<()> { let display = test_tpcds_query("q54").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [segment@0 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[segment@0 ASC], preserve_partitioning=[true] @@ -6742,7 +6604,7 @@ mod tests { │ AggregateExec: mode=FinalPartitioned, gby=[segment@0 as segment], aggr=[count(Int64(1))] │ RepartitionExec: partitioning=Hash([segment@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[segment@0 as segment], aggr=[count(Int64(1))] - │ ProjectionExec: expr=[CAST(round(sum(store_sales.ss_ext_sales_price)@1 / Some(50),20,0) AS Int32) as segment] + │ ProjectionExec: expr=[CAST(round(sum(store_sales.ss_ext_sales_price)@1 / 50) AS Int32) as segment] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] @@ -6765,12 +6627,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@1, d_date_sk@0)], projection=[c_customer_sk@0, ss_ext_sales_price@2, d_month_seq@4] │ CoalescePartitionsExec │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=1, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -6782,12 +6643,11 @@ mod tests { │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(1)@0 as date_dim.d_month_seq + Int64(1)], aggr=[] │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 1 as date_dim.d_month_seq + Int64(1)] │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -6796,15 +6656,15 @@ mod tests { │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@1)], projection=[c_customer_sk@3, ss_sold_date_sk@5, ss_ext_sales_price@6, ca_county@1, ca_state@2] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_sk@0, c_current_addr_sk@1, ss_sold_date_sk@2, ss_ext_sales_price@4] │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -6814,13 +6674,13 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_county, s_state, CAST(s_state@24 AS Utf8View) as CAST(store.s_state AS Utf8View)], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_county, s_state, CAST(s_state@24 AS Utf8View) as CAST(store.s_state AS Utf8View)], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=2, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -6830,63 +6690,59 @@ mod tests { ┌───── Stage 8 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_sk@0, c_current_addr_sk@1], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk], aggr=[] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_sk@0, c_customer_sk@0)], projection=[c_customer_sk@1, c_current_addr_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(customer_sk@0, c_customer_sk@0)] │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sold_date_sk@0)], projection=[customer_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, sold_date_sk@0)], projection=[customer_sk@1] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@2)], projection=[sold_date_sk@1, customer_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_item_sk@0, item_sk@2)], projection=[sold_date_sk@0, customer_sk@1] │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@3 as customer_sk, cs_item_sk@15 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@3 as customer_sk, cs_item_sk@15 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@3 as customer_sk, cs_item_sk@15 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@3 as customer_sk, cs_item_sk@15 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@4 as customer_sk, ws_item_sk@3 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@4 as customer_sk, ws_item_sk@3 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@4 as customer_sk, ws_item_sk@3 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@4 as customer_sk, ws_item_sk@3 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 12 AND d_year@1 = 1998, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@2 = Women AND i_class@1 = maternity, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([date_dim.d_month_seq + Int64(3)@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(3)@0 as date_dim.d_month_seq + Int64(3)], aggr=[] │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 3 as date_dim.d_month_seq + Int64(3)] │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] └────────────────────────────────────────────────── "); Ok(()) @@ -6894,16 +6750,17 @@ mod tests { #[tokio::test] async fn test_tpcds_55() -> Result<()> { let display = test_tpcds_query("q55").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ext_price@2 DESC, brand_id@0 ASC NULLS LAST], fetch=100 │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[ext_price@2 DESC, brand_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, sum(store_sales.ss_ext_sales_price)@2 as ext_price] - │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(store_sales.ss_ext_sales_price)@2 DESC, i_brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1], 6), input_partitions=3 @@ -6915,30 +6772,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manager_id@3 = 28, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] └────────────────────────────────────────────────── "); Ok(()) @@ -6946,25 +6801,26 @@ mod tests { #[tokio::test] async fn test_tpcds_56() -> Result<()> { let display = test_tpcds_query("q56").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [total_sales@1 ASC, i_item_id@0 ASC], fetch=100 │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] - │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] - │ InterleaveExec - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(tmp1.total_sales)@1 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -6974,7 +6830,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_ext_sales_price@3, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_item_sk@1, ss_ext_sales_price@3] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -6982,48 +6838,45 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7034,22 +6887,20 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7060,30 +6911,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7094,22 +6943,20 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7120,30 +6967,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -7156,26 +7001,27 @@ mod tests { │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], fetch=100 │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, avg_monthly_sales@5 as avg_monthly_sales, sum_sales@6 as sum_sales, sum_sales@7 as psum, sum_sales@8 as nsum] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lead.rn - Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, avg_monthly_sales@6, sum_sales@5, sum_sales@8, sum_sales@13] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lead.rn - Decimal128(1,20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, avg_monthly_sales@6, sum_sales@5, sum_sales@8, sum_sales@13] │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, CAST(v1.rn AS Decimal128(21, 0))@9], 3), input_partitions=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum_sales@5 as sum_sales, avg_monthly_sales@6 as avg_monthly_sales, rn@7 as rn, sum_sales@8 as sum_sales, CAST(rn@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@8, v1_lag.rn + Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, rn@7, sum_sales@12] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@8, v1_lag.rn + Decimal128(1,20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, rn@7, sum_sales@12] │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, CAST(v1.rn AS Decimal128(21, 0))@8], 3), input_partitions=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum(catalog_sales.cs_sales_price)@5 as sum_sales, avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] - │ FilterExec: d_year@3 = 1999 AND avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),21,6 AND CASE WHEN avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),21,6 THEN abs(sum(catalog_sales.cs_sales_price)@5 - avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6) / avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 END > Some(1000000000),32,10 - │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 - │ BoundedWindowAggExec: wdw=[avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(21, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, v1_lag.rn + Decimal128(Some(1),20,0)@5], 3), input_partitions=3 - │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ FilterExec: __common_expr_3@0 AND d_year@4 = 1999 AND CASE WHEN __common_expr_3@0 THEN abs(sum(catalog_sales.cs_sales_price)@6 - avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7) / avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 END > 0.1000000000, projection=[i_category@1, i_brand@2, cc_name@3, d_year@4, d_moy@5, sum(catalog_sales.cs_sales_price)@6, avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8] + │ ProjectionExec: expr=[avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > 0.000000 as __common_expr_3, i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum(catalog_sales.cs_sales_price)@5 as sum(catalog_sales.cs_sales_price), avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(21, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, v1_lag.rn + Decimal128(1,20,0)@5], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) + 1 as v1_lag.rn + Decimal128(1,20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, v1_lead.rn - Decimal128(Some(1),20,0)@5], 3), input_partitions=3 - │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, v1_lead.rn - Decimal128(1,20,0)@5], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) - 1 as v1_lead.rn - Decimal128(1,20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -7196,12 +7042,12 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -7214,20 +7060,19 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 @@ -7245,12 +7090,12 @@ mod tests { │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -7263,20 +7108,19 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 @@ -7294,12 +7138,12 @@ mod tests { │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -7312,20 +7156,19 @@ mod tests { ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 13 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── "#); Ok(()) @@ -7333,15 +7176,15 @@ mod tests { #[tokio::test] async fn test_tpcds_58() -> Result<()> { let display = test_tpcds_query("q58").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [item_id@0 ASC], fetch=100 │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 18 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[item_id@0 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, ss_item_rev@2 / __common_expr_7@0 * Some(100),20,0 as ss_dev, cs_item_rev@3 as cs_item_rev, cs_item_rev@3 / __common_expr_7@0 * Some(100),20,0 as cs_dev, ws_item_rev@4 as ws_item_rev, ws_item_rev@4 / __common_expr_7@0 * Some(100),20,0 as ws_dev, __common_expr_7@0 as average] - │ ProjectionExec: expr=[(ss_item_rev@0 + cs_item_rev@1 + ws_item_rev@2) / Some(3),20,0 as __common_expr_7, item_id@3 as item_id, ss_item_rev@0 as ss_item_rev, cs_item_rev@1 as cs_item_rev, ws_item_rev@2 as ws_item_rev] + │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, ss_item_rev@2 / __common_expr_7@0 * 100 as ss_dev, cs_item_rev@3 as cs_item_rev, cs_item_rev@3 / __common_expr_7@0 * 100 as cs_dev, ws_item_rev@4 as ws_item_rev, ws_item_rev@4 / __common_expr_7@0 * 100 as ws_dev, __common_expr_7@0 as average] + │ ProjectionExec: expr=[(ss_item_rev@0 + cs_item_rev@1 + ws_item_rev@2) / 3 as __common_expr_7, item_id@3 as item_id, ss_item_rev@0 as ss_item_rev, cs_item_rev@1 as cs_item_rev, ws_item_rev@2 as ws_item_rev] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], filter=CAST(ss_item_rev@0 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ss_item_rev@0 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)), projection=[ss_item_rev@3, cs_item_rev@4, ws_item_rev@1, item_id@2] │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 @@ -7366,53 +7209,51 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_ext_sales_price@3, i_item_id@4, d_date@1] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@2, ws_ext_sales_price@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 @@ -7428,53 +7269,51 @@ mod tests { │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ext_sales_price@3, i_item_id@4, d_date@1] │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@2, cs_ext_sales_price@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 9 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7484,53 +7323,51 @@ mod tests { │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_ext_sales_price@3, i_item_id@4, d_date@1] │ CoalescePartitionsExec - │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@2, ss_ext_sales_price@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 15 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 16 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -7557,7 +7394,7 @@ mod tests { │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 @@ -7570,18 +7407,17 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@0 >= 1212 AND d_month_seq@0 <= 1223, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -7593,33 +7429,32 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_store_sk@4, ss_sales_price@5, d_week_seq@1, d_day_name@2] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@0 >= 1224 AND d_month_seq@0 <= 1235, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -7631,23 +7466,23 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE None,7,2 END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Sunday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Monday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Tuesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Wednesday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Thursday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Friday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Saturday THEN store_sales.ss_sales_price ELSE NULL END) as sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_store_sk@4, ss_sales_price@5, d_week_seq@1, d_day_name@2] │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "#); Ok(()) @@ -7655,25 +7490,26 @@ mod tests { #[tokio::test] async fn test_tpcds_60() -> Result<()> { let display = test_tpcds_query("q60").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] - │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] - │ InterleaveExec - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7683,7 +7519,7 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_ext_sales_price@3, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_item_sk@1, ss_ext_sales_price@3] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -7691,48 +7527,45 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7743,22 +7576,20 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7769,30 +7600,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -7803,22 +7632,20 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7829,30 +7656,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] └────────────────────────────────────────────────── "); Ok(()) @@ -7863,7 +7688,7 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortExec: TopK(fetch=100), expr=[total@1 ASC NULLS LAST], preserve_partitioning=[false] - │ ProjectionExec: expr=[promotions@0 as promotions, total@1 as total, CAST(promotions@0 AS Decimal128(15, 4)) / CAST(total@1 AS Decimal128(15, 4)) * Some(100),20,0 as promotional_sales.promotions / all_sales.total * Int64(100)] + │ ProjectionExec: expr=[promotions@0 as promotions, total@1 as total, CAST(promotions@0 AS Decimal128(15, 4)) / CAST(total@1 AS Decimal128(15, 4)) * 100 as promotional_sales.promotions / all_sales.total * Int64(100)] │ CrossJoinExec │ CoalescePartitionsExec │ BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 @@ -7887,32 +7712,29 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, c_customer_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7926,20 +7748,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -7953,13 +7774,13 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: s_gmt_offset@1 = Some(-500),5,2, projection=[s_store_sk@0] + │ FilterExec: s_gmt_offset@1 = -5.00, projection=[s_store_sk@0] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] @@ -7972,32 +7793,29 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, c_customer_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] │ CoalescePartitionsExec │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_gmt_offset@1 = Some(-500),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),5,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),5,2)] + │ FilterExec: ca_gmt_offset@1 = -5.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -5.00, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -5.00 AND -5.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-5.00)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -8008,30 +7826,29 @@ mod tests { │ CoalescePartitionsExec │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: s_gmt_offset@1 = Some(-500),5,2, projection=[s_store_sk@0] + │ FilterExec: s_gmt_offset@1 = -5.00, projection=[s_store_sk@0] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),5,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),5,2 AND Some(-500),5,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),5,2)] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = -5.00, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= -5.00 AND -5.00 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (-5.00)] └────────────────────────────────────────────────── "); Ok(()) @@ -8039,16 +7856,17 @@ mod tests { #[tokio::test] async fn test_tpcds_62() -> Result<()> { let display = test_tpcds_query("q62").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], fetch=100 │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@3 as 30 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@4 as 31-60 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@5 as 61-90 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@6 as 91-120 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@7 as >120 days] - │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name], aggr=[sum(CASE WHEN __common_expr_1 <= 30 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 30 AND __common_expr_1 <= 60 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 60 AND __common_expr_1 <= 90 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 90 AND __common_expr_1 <= 120 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 120 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name], aggr=[sum(CASE WHEN __common_expr_1 <= 30 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 30 AND __common_expr_1 <= 60 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 60 AND __common_expr_1 <= 90 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 90 AND __common_expr_1 <= 120 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN __common_expr_1 > 120 THEN 1 ELSE 0 END) as sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([w_substr@0, sm_type@1, web_name@2], 6), input_partitions=3 @@ -8067,20 +7885,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8115,9 +7932,9 @@ mod tests { assert_snapshot!(display, @r#" ┌───── DistributedExec │ SortPreservingMergeExec: [i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], fetch=100 - │ SortExec: TopK(fetch=100), expr=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST] - │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_monthly_sales] - │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),21,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),32,10 + │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_monthly_sales] + │ SortExec: TopK(fetch=100), expr=[i_manager_id@0 ASC NULLS LAST, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 ASC NULLS LAST, sum(store_sales.ss_sales_price)@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[i_manager_id@0 ASC NULLS LAST, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 ASC NULLS LAST] + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > 0.000000 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > 0.1000000000 │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(21, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] │ SortExec: expr=[i_manager_id@0 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -8141,10 +7958,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8157,22 +7974,20 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_moy@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manager_id@4] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] └────────────────────────────────────────────────── "#); Ok(()) @@ -8180,7 +7995,7 @@ mod tests { #[tokio::test] async fn test_tpcds_64() -> Result<()> { let display = test_tpcds_query("q64").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [product_name@0 ASC NULLS LAST, store_name@1 ASC NULLS LAST, cnt@20 ASC NULLS LAST, s11@13 ASC NULLS LAST, s12@16 ASC NULLS LAST] │ [Stage 52] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -8207,13 +8022,12 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, d_year@7, d_year@8, s_store_name@5, s_zip@6, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] │ CoalescePartitionsExec │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),7,2 AND i_current_price@1 <= Some(7400),7,2, projection=[i_item_sk@0, i_product_name@3] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= 65.00 AND i_current_price@1 <= 74.00, projection=[i_item_sk@0, i_product_name@3] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] └────────────────────────────────────────────────── ┌───── Stage 24 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8221,10 +8035,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 23 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -8233,10 +8047,10 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@7)], projection=[ss_item_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, d_year@13, d_year@14, hd_income_band_sk@15, hd_income_band_sk@16, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20, ca_street_number@1, ca_street_name@2, ca_city@3, ca_zip@4] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_item_sk@5, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_addr_sk@13, d_year@14, d_year@15, hd_income_band_sk@16, hd_income_band_sk@17, ca_street_number@1, ca_street_name@2, ca_city@3, ca_zip@4] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, c_current_hdemo_sk@8)], projection=[ss_item_sk@2, ss_addr_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_addr_sk@11, d_year@12, d_year@13, hd_income_band_sk@14, hd_income_band_sk@1] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -8246,8 +8060,8 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_promo_sk@0, ss_promo_sk@3)], projection=[ss_item_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, c_current_cdemo_sk@10)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@13, c_current_addr_sk@14, d_year@15, d_year@16] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_current_cdemo_sk@10, cd_demo_sk@0)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 @@ -8258,21 +8072,21 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8285,10 +8099,10 @@ mod tests { ┌───── Stage 5 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8298,80 +8112,64 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:..]]}, projection=[p_promo_sk], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet:..]]}, projection=[p_promo_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 22 ── tasks=4, partitions=12 + ┌───── Stage 21 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_current_cdemo_sk@10], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], projection=[ss_item_sk@2, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, d_year@16, d_year@17, cd_marital_status@1] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ss_cdemo_sk@1, cd_demo_sk@0)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 21 ── tasks=4, partitions=12 + ┌───── Stage 19 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, c_first_shipto_date_sk@14)], projection=[ss_item_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, d_year@17, d_year@1] │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, c_first_sales_date_sk@15)], projection=[ss_item_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, d_year@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_item_sk@6, ss_cdemo_sk@8, ss_hdemo_sk@9, ss_addr_sk@10, ss_promo_sk@11, ss_wholesale_cost@12, ss_list_price@13, ss_coupon_amt@14, d_year@15, s_store_name@16, s_zip@17, c_current_cdemo_sk@1, c_current_hdemo_sk@2, c_current_addr_sk@3, c_first_shipto_date_sk@4, c_first_sales_date_sk@5] - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=2 + ┌───── Stage 9 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 20 ── tasks=4, partitions=12 + ┌───── Stage 18 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@5)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_promo_sk@9, ss_wholesale_cost@10, ss_list_price@11, ss_coupon_amt@12, d_year@13, s_store_name@1, s_zip@2] │ CoalescePartitionsExec - │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@10, ss_list_price@11, ss_coupon_amt@12, d_year@1] │ CoalescePartitionsExec - │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(cs_item_sk@0, ss_item_sk@1)] │ CoalescePartitionsExec - │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] - │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=32 + ┌───── Stage 10 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet @@ -8379,30 +8177,29 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=4, partitions=48 + ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] └────────────────────────────────────────────────── - ┌───── Stage 17 ── tasks=2, partitions=24 + ┌───── Stage 15 ── tasks=2, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > 2 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=4, partitions=6 + ┌───── Stage 14 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cs_item_sk@5, cs_ext_list_price@7, cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=12 + ┌───── Stage 12 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet @@ -8410,15 +8207,15 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=4, partitions=12 + ┌───── Stage 13 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 18 ── tasks=4, partitions=12 + ┌───── Stage 16 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 12), input_partitions=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet @@ -8426,27 +8223,42 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 19 ── tasks=4, partitions=12 + ┌───── Stage 17 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 20 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 22 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── ┌───── Stage 51 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_product_name@0, i_item_sk@1, s_store_name@2, s_zip@3, ca_street_number@4, ca_street_name@5, ca_city@6, ca_zip@7, ca_street_number@8, ca_street_name@9, ca_city@10, ca_zip@11, d_year@12, d_year@13, d_year@14], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_product_name@17 as i_product_name, i_item_sk@16 as i_item_sk, s_store_name@6 as s_store_name, s_zip@7 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, d_year@3 as d_year, d_year@4 as d_year, d_year@5 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, d_year@7, d_year@8, s_store_name@5, s_zip@6, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] │ CoalescePartitionsExec │ [Stage 50] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),7,2 AND i_current_price@1 <= Some(7400),7,2, projection=[i_item_sk@0, i_product_name@3] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),7,2 AND i_current_price@5 <= Some(7400),7,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),7,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),7,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= 65.00 AND i_current_price@1 <= 74.00, projection=[i_item_sk@0, i_product_name@3] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= 65.00 AND i_current_price@5 <= 74.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= 65.00 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= 74.00, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] └────────────────────────────────────────────────── ┌───── Stage 50 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8454,10 +8266,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 49] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 49 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -8466,10 +8278,10 @@ mod tests { │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@7)], projection=[ss_item_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, d_year@13, d_year@14, hd_income_band_sk@15, hd_income_band_sk@16, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20, ca_street_number@1, ca_street_name@2, ca_city@3, ca_zip@4] │ CoalescePartitionsExec - │ [Stage 28] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 28] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_item_sk@5, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_addr_sk@13, d_year@14, d_year@15, hd_income_band_sk@16, hd_income_band_sk@17, ca_street_number@1, ca_street_name@2, ca_city@3, ca_zip@4] │ CoalescePartitionsExec - │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, c_current_hdemo_sk@8)], projection=[ss_item_sk@2, ss_addr_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_addr_sk@11, d_year@12, d_year@13, hd_income_band_sk@14, hd_income_band_sk@1] │ CoalescePartitionsExec │ [Stage 30] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -8479,8 +8291,8 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_promo_sk@0, ss_promo_sk@3)], projection=[ss_item_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] │ CoalescePartitionsExec │ [Stage 32] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, c_current_cdemo_sk@10)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@13, c_current_addr_sk@14, d_year@15, d_year@16] - │ [Stage 33] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_current_cdemo_sk@10, cd_demo_sk@0)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ [Stage 47] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 48] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 27 ── tasks=4, partitions=32 @@ -8491,21 +8303,21 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet:..]]}, projection=[ib_income_band_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 28 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 28 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 29 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 29 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 30 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8518,10 +8330,10 @@ mod tests { ┌───── Stage 31 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 32 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8531,80 +8343,64 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:..]]}, projection=[p_promo_sk], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet:..]]}, projection=[p_promo_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 33 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 48 ── tasks=4, partitions=12 + ┌───── Stage 47 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_current_cdemo_sk@10], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cd_demo_sk@0, ss_cdemo_sk@1)], projection=[ss_item_sk@2, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, d_year@16, d_year@17, cd_marital_status@1] - │ [Stage 34] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 47] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ss_cdemo_sk@1, cd_demo_sk@0)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 46] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 34 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 47 ── tasks=4, partitions=12 + ┌───── Stage 45 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, c_first_shipto_date_sk@14)], projection=[ss_item_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, d_year@17, d_year@1] │ CoalescePartitionsExec - │ [Stage 35] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 33] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, c_first_sales_date_sk@15)], projection=[ss_item_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_promo_sk@6, ss_wholesale_cost@7, ss_list_price@8, ss_coupon_amt@9, d_year@10, s_store_name@11, s_zip@12, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, d_year@1] │ CoalescePartitionsExec - │ [Stage 36] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 34] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_item_sk@6, ss_cdemo_sk@8, ss_hdemo_sk@9, ss_addr_sk@10, ss_promo_sk@11, ss_wholesale_cost@12, ss_list_price@13, ss_coupon_amt@14, d_year@15, s_store_name@16, s_zip@17, c_current_cdemo_sk@1, c_current_hdemo_sk@2, c_current_addr_sk@3, c_first_shipto_date_sk@4, c_first_sales_date_sk@5] - │ [Stage 37] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 46] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 35] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 35 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 33 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 36 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 34 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 37 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=2 + ┌───── Stage 35 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 46 ── tasks=4, partitions=12 + ┌───── Stage 44 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@5)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_promo_sk@9, ss_wholesale_cost@10, ss_list_price@11, ss_coupon_amt@12, d_year@13, s_store_name@1, s_zip@2] │ CoalescePartitionsExec - │ [Stage 38] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 36] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@10, ss_list_price@11, ss_coupon_amt@12, d_year@1] │ CoalescePartitionsExec - │ [Stage 39] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ [Stage 37] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(cs_item_sk@0, ss_item_sk@1)] │ CoalescePartitionsExec - │ [Stage 43] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ [Stage 41] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] - │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 43] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 38 ── tasks=4, partitions=32 + ┌───── Stage 36 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet @@ -8612,30 +8408,29 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet:..]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 39 ── tasks=4, partitions=48 + ┌───── Stage 37 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 43 ── tasks=2, partitions=24 + ┌───── Stage 41 ── tasks=2, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > 2 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 40] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 42 ── tasks=4, partitions=6 + ┌───── Stage 40 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cs_item_sk@5, cs_ext_list_price@7, cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4] - │ [Stage 40] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 41] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 40 ── tasks=4, partitions=12 + ┌───── Stage 38 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet @@ -8643,15 +8438,15 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 41 ── tasks=4, partitions=12 + ┌───── Stage 39 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 44 ── tasks=4, partitions=12 + ┌───── Stage 42 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 12), input_partitions=3 │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet @@ -8659,28 +8454,44 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 45 ── tasks=4, partitions=12 + ┌───── Stage 43 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 46 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 48 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_65() -> Result<()> { let display = test_tpcds_query("q65").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [s_store_name@0 ASC, i_item_desc@1 ASC], fetch=100 │ [Stage 9] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=3 │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC, i_item_desc@1 ASC], preserve_partitioning=[true] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ss_store_sk@1, ss_store_sk@0)], filter=CAST(revenue@0 AS Decimal128(30, 15)) <= CAST(0.1 * CAST(ave@1 AS Float64) AS Decimal128(30, 15)), projection=[s_store_name@0, i_item_desc@3, revenue@2, i_current_price@4, i_wholesale_cost@5, i_brand@6] + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ss_store_sk@1, ss_store_sk@0)], filter=CAST(revenue@0 AS Decimal128(30, 15)) <= CAST(0.1 * CAST(ave@1 AS Float64) AS Decimal128(30, 15)), projection=[s_store_name@0, i_item_desc@3, revenue@2, i_current_price@4, i_wholesale_cost@5, i_brand@6] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, avg(sa.revenue)@1 as ave] │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(sa.revenue)] @@ -8690,7 +8501,7 @@ mod tests { │ RepartitionExec: partitioning=Hash([ss_store_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@2)], projection=[s_store_name@5, ss_store_sk@6, revenue@8, i_item_desc@1, i_current_price@2, i_wholesale_cost@3, i_brand@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@0)], projection=[s_store_name@1, ss_store_sk@2, ss_item_sk@3, revenue@4] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -8698,13 +8509,13 @@ mod tests { │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8721,20 +8532,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=12 │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 12), input_partitions=3 @@ -8750,20 +8560,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -8774,8 +8583,8 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortPreservingMergeExec: [w_warehouse_name@0 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, sum(x.jan_sales)@8 as jan_sales, sum(x.feb_sales)@9 as feb_sales, sum(x.mar_sales)@10 as mar_sales, sum(x.apr_sales)@11 as apr_sales, sum(x.may_sales)@12 as may_sales, sum(x.jun_sales)@13 as jun_sales, sum(x.jul_sales)@14 as jul_sales, sum(x.aug_sales)@15 as aug_sales, sum(x.sep_sales)@16 as sep_sales, sum(x.oct_sales)@17 as oct_sales, sum(x.nov_sales)@18 as nov_sales, sum(x.dec_sales)@19 as dec_sales, sum(x.jan_sales / x.w_warehouse_sq_ft)@20 as jan_sales_per_sq_foot, sum(x.feb_sales / x.w_warehouse_sq_ft)@21 as feb_sales_per_sq_foot, sum(x.mar_sales / x.w_warehouse_sq_ft)@22 as mar_sales_per_sq_foot, sum(x.apr_sales / x.w_warehouse_sq_ft)@23 as apr_sales_per_sq_foot, sum(x.may_sales / x.w_warehouse_sq_ft)@24 as may_sales_per_sq_foot, sum(x.jun_sales / x.w_warehouse_sq_ft)@25 as jun_sales_per_sq_foot, sum(x.jul_sales / x.w_warehouse_sq_ft)@26 as jul_sales_per_sq_foot, sum(x.aug_sales / x.w_warehouse_sq_ft)@27 as aug_sales_per_sq_foot, sum(x.sep_sales / x.w_warehouse_sq_ft)@28 as sep_sales_per_sq_foot, sum(x.oct_sales / x.w_warehouse_sq_ft)@29 as oct_sales_per_sq_foot, sum(x.nov_sales / x.w_warehouse_sq_ft)@30 as nov_sales_per_sq_foot, sum(x.dec_sales / x.w_warehouse_sq_ft)@31 as dec_sales_per_sq_foot, sum(x.jan_net)@32 as jan_net, sum(x.feb_net)@33 as feb_net, sum(x.mar_net)@34 as mar_net, sum(x.apr_net)@35 as apr_net, sum(x.may_net)@36 as may_net, sum(x.jun_net)@37 as jun_net, sum(x.jul_net)@38 as jul_net, sum(x.aug_net)@39 as aug_net, sum(x.sep_net)@40 as sep_net, sum(x.oct_net)@41 as oct_net, sum(x.nov_net)@42 as nov_net, sum(x.dec_net)@43 as dec_net] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, sum(x.jan_sales)@8 as jan_sales, sum(x.feb_sales)@9 as feb_sales, sum(x.mar_sales)@10 as mar_sales, sum(x.apr_sales)@11 as apr_sales, sum(x.may_sales)@12 as may_sales, sum(x.jun_sales)@13 as jun_sales, sum(x.jul_sales)@14 as jul_sales, sum(x.aug_sales)@15 as aug_sales, sum(x.sep_sales)@16 as sep_sales, sum(x.oct_sales)@17 as oct_sales, sum(x.nov_sales)@18 as nov_sales, sum(x.dec_sales)@19 as dec_sales, sum(x.jan_sales / x.w_warehouse_sq_ft)@20 as jan_sales_per_sq_foot, sum(x.feb_sales / x.w_warehouse_sq_ft)@21 as feb_sales_per_sq_foot, sum(x.mar_sales / x.w_warehouse_sq_ft)@22 as mar_sales_per_sq_foot, sum(x.apr_sales / x.w_warehouse_sq_ft)@23 as apr_sales_per_sq_foot, sum(x.may_sales / x.w_warehouse_sq_ft)@24 as may_sales_per_sq_foot, sum(x.jun_sales / x.w_warehouse_sq_ft)@25 as jun_sales_per_sq_foot, sum(x.jul_sales / x.w_warehouse_sq_ft)@26 as jul_sales_per_sq_foot, sum(x.aug_sales / x.w_warehouse_sq_ft)@27 as aug_sales_per_sq_foot, sum(x.sep_sales / x.w_warehouse_sq_ft)@28 as sep_sales_per_sq_foot, sum(x.oct_sales / x.w_warehouse_sq_ft)@29 as oct_sales_per_sq_foot, sum(x.nov_sales / x.w_warehouse_sq_ft)@30 as nov_sales_per_sq_foot, sum(x.dec_sales / x.w_warehouse_sq_ft)@31 as dec_sales_per_sq_foot, sum(x.jan_net)@32 as jan_net, sum(x.feb_net)@33 as feb_net, sum(x.mar_net)@34 as mar_net, sum(x.apr_net)@35 as apr_net, sum(x.may_net)@36 as may_net, sum(x.jun_net)@37 as jun_net, sum(x.jul_net)@38 as jul_net, sum(x.aug_net)@39 as aug_net, sum(x.sep_net)@40 as sep_net, sum(x.oct_net)@41 as oct_net, sum(x.nov_net)@42 as nov_net, sum(x.dec_net)@43 as dec_net] + │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_], aggr=[sum(x.jan_sales), sum(x.feb_sales), sum(x.mar_sales), sum(x.apr_sales), sum(x.may_sales), sum(x.jun_sales), sum(x.jul_sales), sum(x.aug_sales), sum(x.sep_sales), sum(x.oct_sales), sum(x.nov_sales), sum(x.dec_sales), sum(x.jan_sales / x.w_warehouse_sq_ft), sum(x.feb_sales / x.w_warehouse_sq_ft), sum(x.mar_sales / x.w_warehouse_sq_ft), sum(x.apr_sales / x.w_warehouse_sq_ft), sum(x.may_sales / x.w_warehouse_sq_ft), sum(x.jun_sales / x.w_warehouse_sq_ft), sum(x.jul_sales / x.w_warehouse_sq_ft), sum(x.aug_sales / x.w_warehouse_sq_ft), sum(x.sep_sales / x.w_warehouse_sq_ft), sum(x.oct_sales / x.w_warehouse_sq_ft), sum(x.nov_sales / x.w_warehouse_sq_ft), sum(x.dec_sales / x.w_warehouse_sq_ft), sum(x.jan_net), sum(x.feb_net), sum(x.mar_net), sum(x.apr_net), sum(x.may_net), sum(x.jun_net), sum(x.jul_net), sum(x.aug_net), sum(x.sep_net), sum(x.oct_net), sum(x.nov_net), sum(x.dec_net)] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── @@ -8785,15 +8594,15 @@ mod tests { │ ProjectionExec: expr=[CAST(w_warehouse_sq_ft@1 AS Decimal128(10, 0)) as __common_expr_1, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, jan_sales@8 as jan_sales, feb_sales@9 as feb_sales, mar_sales@10 as mar_sales, apr_sales@11 as apr_sales, may_sales@12 as may_sales, jun_sales@13 as jun_sales, jul_sales@14 as jul_sales, aug_sales@15 as aug_sales, sep_sales@16 as sep_sales, oct_sales@17 as oct_sales, nov_sales@18 as nov_sales, dec_sales@19 as dec_sales, jan_net@20 as jan_net, feb_net@21 as feb_net, mar_net@22 as mar_net, apr_net@23 as apr_net, may_net@24 as may_net, jun_net@25 as jun_net, jul_net@26 as jul_net, aug_net@27 as aug_net, sep_net@28 as sep_net, oct_net@29 as oct_net, nov_net@30 as nov_net, dec_net@31 as dec_net] │ InterleaveExec │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@30 as dec_net] - │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN __common_expr_2 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_2 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN __common_expr_2 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_2 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@30 as dec_net] - │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN __common_expr_2 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_2 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN __common_expr_2 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_2 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_3 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_4 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_5 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_6 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_7 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_8 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_9 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_10 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_11 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_12 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_13 THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) │ ProjectionExec: expr=[d_moy@0 = 1 as __common_expr_2, d_moy@0 = 2 as __common_expr_3, d_moy@0 = 3 as __common_expr_4, d_moy@0 = 4 as __common_expr_5, d_moy@0 = 5 as __common_expr_6, d_moy@0 = 6 as __common_expr_7, d_moy@0 = 7 as __common_expr_8, d_moy@0 = 8 as __common_expr_9, d_moy@0 = 9 as __common_expr_10, d_moy@0 = 10 as __common_expr_11, d_moy@0 = 11 as __common_expr_12, d_moy@0 = 12 as __common_expr_13, ws_quantity@1 as ws_quantity, ws_ext_sales_price@2 as ws_ext_sales_price, ws_net_paid@3 as ws_net_paid, w_warehouse_name@4 as w_warehouse_name, w_warehouse_sq_ft@5 as w_warehouse_sq_ft, w_city@6 as w_city, w_county@7 as w_county, w_state@8 as w_state, w_country@9 as w_country, d_year@10 as d_year] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sm_ship_mode_sk@0, ws_ship_mode_sk@0)], projection=[d_moy@12, ws_quantity@2, ws_ext_sales_price@3, ws_net_paid@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11] │ CoalescePartitionsExec @@ -8808,10 +8617,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -8826,22 +8635,20 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8853,7 +8660,7 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Some(0),22,2 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_14 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_15 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_16 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_17 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_18 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_19 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_20 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_21 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_22 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_23 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_24 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN __common_expr_25 THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE 0.00 END) as sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) │ ProjectionExec: expr=[d_moy@0 = 1 as __common_expr_14, d_moy@0 = 2 as __common_expr_15, d_moy@0 = 3 as __common_expr_16, d_moy@0 = 4 as __common_expr_17, d_moy@0 = 5 as __common_expr_18, d_moy@0 = 6 as __common_expr_19, d_moy@0 = 7 as __common_expr_20, d_moy@0 = 8 as __common_expr_21, d_moy@0 = 9 as __common_expr_22, d_moy@0 = 10 as __common_expr_23, d_moy@0 = 11 as __common_expr_24, d_moy@0 = 12 as __common_expr_25, cs_quantity@1 as cs_quantity, cs_sales_price@2 as cs_sales_price, cs_net_paid_inc_tax@3 as cs_net_paid_inc_tax, w_warehouse_name@4 as w_warehouse_name, w_warehouse_sq_ft@5 as w_warehouse_sq_ft, w_city@6 as w_city, w_county@7 as w_county, w_state@8 as w_state, w_country@9 as w_country, d_year@10 as d_year] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sm_ship_mode_sk@0, cs_ship_mode_sk@0)], projection=[d_moy@12, cs_quantity@2, cs_sales_price@3, cs_net_paid_inc_tax@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11] │ CoalescePartitionsExec @@ -8868,10 +8675,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -8886,22 +8693,20 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8920,8 +8725,8 @@ mod tests { assert_snapshot!(display, @r#" ┌───── DistributedExec │ SortPreservingMergeExec: [i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sumsales@8 as sumsales, rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 as rk] + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sumsales@8 as sumsales, rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 as rk] + │ SortExec: TopK(fetch=100), expr=[i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 ASC], preserve_partitioning=[true] │ FilterExec: rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 <= 100 │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, sumsales@8 DESC], preserve_partitioning=[true] @@ -8930,16 +8735,16 @@ mod tests { ┌───── Stage 5 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))@9 as sumsales] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, __grouping_id@8 as __grouping_id], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Some(0),22,2 END) as sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, __grouping_id@8 as __grouping_id], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0.00 END) as sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1, i_brand@2, i_product_name@3, d_year@4, d_qoy@5, d_moy@6, s_store_id@7, __grouping_id@8], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[(NULL as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, s_store_id@4 as s_store_id)], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Some(0),22,2 END) as sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[(NULL as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, s_store_id@4 as s_store_id)], aggr=[sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0.00 END) as sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] │ ProjectionExec: expr=[CAST(ss_sales_price@0 * CAST(ss_quantity@1 AS Decimal128(10, 0)) AS Decimal128(22, 2)) as __common_expr_1, d_year@2 as d_year, d_moy@3 as d_moy, d_qoy@4 as d_qoy, s_store_id@5 as s_store_id, i_brand@6 as i_brand, i_class@7 as i_class, i_category@8 as i_category, i_product_name@9 as i_product_name] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_sales_price@7, ss_quantity@6, d_year@8, d_moy@9, d_qoy@10, s_store_id@11, i_brand@1, i_class@2, i_category@3, i_product_name@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_sk@0, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_sales_price@5, d_year@6, d_moy@7, d_qoy@8, s_store_id@1] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -8947,18 +8752,18 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -8971,12 +8776,11 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0, d_year@2, d_moy@3, d_qoy@4] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "#); Ok(()) @@ -8984,7 +8788,7 @@ mod tests { #[tokio::test] async fn test_tpcds_68() -> Result<()> { let display = test_tpcds_query("q68").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC, ss_ticket_number@4 ASC], fetch=100 │ [Stage 8] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -8993,36 +8797,36 @@ mod tests { │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@5)], filter=bought_city@0 != ca_city@1, projection=[c_last_name@9, c_first_name@8, ca_city@1, bought_city@3, ss_ticket_number@2, extended_price@4, extended_tax@6, list_price@5] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[ss_ticket_number@4, bought_city@6, extended_price@7, list_price@8, extended_tax@9, c_current_addr_sk@1, c_first_name@2, c_last_name@3] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_ext_sales_price)@4 as extended_price, sum(store_sales.ss_ext_list_price)@5 as list_price, sum(store_sales.ss_ext_tax)@6 as extended_tax] │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 12), input_partitions=3 │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@6 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ss_addr_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@3, ss_ticket_number@4, ss_ext_sales_price@5, ss_ext_list_price@6, ss_ext_tax@7, ca_city@1] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_demo_sk@0, ss_hdemo_sk@1)], projection=[ss_customer_sk@1, ss_addr_sk@3, ss_ticket_number@4, ss_ext_sales_price@5, ss_ext_list_price@6, ss_ext_tax@7] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -9033,18 +8837,18 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -9069,12 +8873,11 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -9085,128 +8888,121 @@ mod tests { assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], fetch=100 - │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 13 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], preserve_partitioning=[true] + ┌───── Stage 12 ── tasks=2, partitions=3 + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@5 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@5 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@5 as cnt3] - │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@3 ASC NULLS LAST, cd_credit_rating@4 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=6 + ┌───── Stage 11 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] │ HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_customer_sk@0, cs_ship_customer_sk@0)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=12 + ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_customer_sk@0, ws_bill_customer_sk@0)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(ws_bill_customer_sk@0, c_customer_sk@0)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(c_customer_sk@0, ss_customer_sk@0)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@1] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_state@1 = KY OR ca_state@1 = GA OR ca_state@1 = NM, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@2] + ┌───── Stage 5 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, cd_demo_sk@0)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 + ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ FilterExec: ca_state@1 = KY OR ca_state@1 = GA OR ca_state@1 = NM, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@2] + ┌───── Stage 7 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 + ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=12 + ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_ship_customer_sk@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@2] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=48 + ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -9298,14 +9094,14 @@ mod tests { #[tokio::test] async fn test_tpcds_71() -> Result<()> { let display = test_tpcds_query("q71").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [ext_price@4 DESC, brand_id@0 ASC, t_hour@2 ASC] │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=2, partitions=3 - │ SortExec: expr=[ext_price@4 DESC, brand_id@0 ASC, t_hour@2 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, sum(tmp.ext_price)@4 as ext_price] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, sum(tmp.ext_price)@4 as ext_price] + │ SortExec: expr=[sum(tmp.ext_price)@4 DESC, i_brand_id@1 ASC, t_hour@2 ASC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, t_hour@2 as t_hour, t_minute@3 as t_minute], aggr=[sum(tmp.ext_price)] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -9324,70 +9120,65 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[cs_ext_sales_price@0 as ext_price, cs_item_sk@1 as sold_item_sk, cs_sold_time_sk@2 as time_sk] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ext_sales_price@4, cs_item_sk@3, cs_sold_time_sk@2] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible │ ProjectionExec: expr=[ss_ext_sales_price@0 as ext_price, ss_item_sk@1 as sold_item_sk, ss_sold_time_sk@2 as time_sk] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_ext_sales_price@4, ss_item_sk@3, ss_sold_time_sk@2] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_meal_time@3 = breakfast OR t_meal_time@3 = dinner, projection=[t_time_sk@0, t_hour@1, t_minute@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1, pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] └────────────────────────────────────────────────── "); Ok(()) @@ -9395,16 +9186,17 @@ mod tests { #[tokio::test] async fn test_tpcds_72() -> Result<()> { let display = test_tpcds_query("q72").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], fetch=100 │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq, sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@3 as no_promo, sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@4 as promo, count(Int64(1))@5 as total_cnt] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[count(Int64(1))@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_desc@0, w_warehouse_name@1, d_week_seq@2], 6), input_partitions=3 @@ -9428,10 +9220,10 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_ship_date_sk@0)], filter=d_date@1 > d_date@0 + IntervalMonthDayNano { months: 0, days: 5, nanoseconds: 0 }, projection=[cs_item_sk@3, cs_promo_sk@4, cs_order_number@5, w_warehouse_name@6, i_item_desc@7, d_week_seq@9] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@4), (d_week_seq@1, d_week_seq@8)], projection=[cs_ship_date_sk@2, cs_item_sk@3, cs_promo_sk@4, cs_order_number@5, w_warehouse_name@7, i_item_desc@8, d_date@9, d_week_seq@10] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_ship_date_sk@4, cs_item_sk@5, cs_promo_sk@6, cs_order_number@7, inv_date_sk@8, w_warehouse_name@9, i_item_desc@10, d_date@1, d_week_seq@2] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -9450,31 +9242,30 @@ mod tests { │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:..]]}, projection=[p_promo_sk], file_type=parquet │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet:..]]}, projection=[p_promo_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 3 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_week_seq], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@3 = 1999, projection=[d_date_sk@0, d_date@1, d_week_seq@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -9493,13 +9284,13 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@4)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@1] │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@8)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@1] │ CoalescePartitionsExec │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 @@ -9507,13 +9298,13 @@ mod tests { │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_desc], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -9526,18 +9317,18 @@ mod tests { ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_item_sk@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -9545,7 +9336,7 @@ mod tests { #[tokio::test] async fn test_tpcds_73() -> Result<()> { let display = test_tpcds_query("q73").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [cnt@5 DESC, c_last_name@0 ASC NULLS LAST] │ [Stage 6] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -9554,19 +9345,19 @@ mod tests { │ SortExec: expr=[cnt@5 DESC, c_last_name@0 ASC NULLS LAST], preserve_partitioning=[true] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_last_name@3, c_first_name@2, c_salutation@1, c_preferred_cust_flag@4, ss_ticket_number@5, cnt@7] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] │ FilterExec: count(Int64(1))@2 >= 1 AND count(Int64(1))@2 <= 5 │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 12), input_partitions=3 @@ -9581,10 +9372,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -9609,12 +9400,11 @@ mod tests { ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -9622,14 +9412,14 @@ mod tests { #[tokio::test] async fn test_tpcds_74() -> Result<()> { let display = test_tpcds_query("q74").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [customer_id@0 ASC], fetch=100 │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),17,2 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),17,2 THEN year_total@1 / year_total@0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > 0.00 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > 0.00 THEN year_total@1 / year_total@0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4] │ CoalescePartitionsExec │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@4 as year_total] @@ -9651,7 +9441,7 @@ mod tests { ┌───── Stage 4 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@1 as year_total] - │ FilterExec: sum(web_sales.ws_net_paid)@1 > Some(0),17,2 + │ FilterExec: sum(web_sales.ws_net_paid)@1 > 0.00 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_net_paid)@4 as sum(web_sales.ws_net_paid)] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -9664,35 +9454,34 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@4, ws_net_paid@6] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 2 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_net_paid)@1 as year_total] - │ FilterExec: sum(store_sales.ss_net_paid)@1 > Some(0),17,2 + │ FilterExec: sum(store_sales.ss_net_paid)@1 > 0.00 │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_net_paid)@4 as sum(store_sales.ss_net_paid)] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -9705,30 +9494,29 @@ mod tests { │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@4, ss_net_paid@6] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=3 @@ -9738,30 +9526,29 @@ mod tests { │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@4, ss_net_paid@6] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (d_year@1 = 2001 OR d_year@1 = 2002) AND d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=3 @@ -9771,30 +9558,29 @@ mod tests { │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@4, ws_net_paid@6] │ CoalescePartitionsExec - │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (d_year@1 = 2001 OR d_year@1 = 2002) AND d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=(d_year@6 = 2001 OR d_year@6 = 2002) AND d_year@6 = 2002, pruning_predicate=(d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1) AND d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 14 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 14 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -9802,12 +9588,12 @@ mod tests { #[tokio::test] async fn test_tpcds_75() -> Result<()> { let display = test_tpcds_query("q75").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], fetch=100 │ SortExec: TopK(fetch=100), expr=[sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[d_year@0 as prev_year, d_year@1 as year_, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, sales_cnt@6 as prev_yr_cnt, sales_cnt@7 as curr_yr_cnt, sales_cnt@7 - sales_cnt@6 as sales_cnt_diff, sales_amt@8 - sales_amt@9 as sales_amt_diff] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_brand_id@1, i_brand_id@1), (i_class_id@2, i_class_id@2), (i_category_id@3, i_category_id@3), (i_manufact_id@4, i_manufact_id@4)], filter=CAST(sales_cnt@0 AS Decimal128(17, 2)) / CAST(sales_cnt@1 AS Decimal128(17, 2)) < Some(900000),23,6, projection=[d_year@7, d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@12, sales_cnt@5, sales_amt@6, sales_amt@13] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_brand_id@1, i_brand_id@1), (i_class_id@2, i_class_id@2), (i_category_id@3, i_category_id@3), (i_manufact_id@4, i_manufact_id@4)], filter=CAST(sales_cnt@0 AS Decimal128(17, 2)) / CAST(sales_cnt@1 AS Decimal128(17, 2)) < 0.900000, projection=[d_year@7, d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@12, sales_cnt@5, sales_amt@6, sales_amt@13] │ RepartitionExec: partitioning=Hash([i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) @@ -9827,17 +9613,17 @@ mod tests { │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(cs_quantity@2 AS Int64) - CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE 0 END as sales_cnt, cs_ext_sales_price@3 - CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(cs_quantity@2 AS Int64) - CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE 0 END as sales_cnt, cs_ext_sales_price@3 - CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(cr_return_quantity@0 AS Int64) as __common_expr_1, CAST(cr_return_amount@1 AS Decimal128(30, 15)) as __common_expr_2, cs_quantity@2 as cs_quantity, cs_ext_sales_price@3 as cs_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cr_return_quantity@11, cr_return_amount@12, cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ss_quantity@2 AS Int64) - CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE 0 END as sales_cnt, ss_ext_sales_price@3 - CASE WHEN __common_expr_4@1 IS NOT NULL THEN __common_expr_4@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ss_quantity@2 AS Int64) - CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE 0 END as sales_cnt, ss_ext_sales_price@3 - CASE WHEN __common_expr_4@1 IS NOT NULL THEN __common_expr_4@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(sr_return_quantity@0 AS Int64) as __common_expr_3, CAST(sr_return_amt@1 AS Decimal128(30, 15)) as __common_expr_4, ss_quantity@2 as ss_quantity, ss_ext_sales_price@3 as ss_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[sr_return_quantity@11, sr_return_amt@12, ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ws_quantity@2 AS Int64) - CASE WHEN __common_expr_5@0 IS NOT NULL THEN __common_expr_5@0 ELSE 0 END as sales_cnt, ws_ext_sales_price@3 - CASE WHEN __common_expr_6@1 IS NOT NULL THEN __common_expr_6@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ws_quantity@2 AS Int64) - CASE WHEN __common_expr_5@0 IS NOT NULL THEN __common_expr_5@0 ELSE 0 END as sales_cnt, ws_ext_sales_price@3 - CASE WHEN __common_expr_6@1 IS NOT NULL THEN __common_expr_6@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(wr_return_quantity@0 AS Int64) as __common_expr_5, CAST(wr_return_amt@1 AS Decimal128(30, 15)) as __common_expr_6, ws_quantity@2 as ws_quantity, ws_ext_sales_price@3 as ws_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[wr_return_quantity@11, wr_return_amt@12, ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -9852,38 +9638,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 6), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([ss_ticket_number@1, ss_item_sk@0], 3), input_partitions=3 @@ -9894,38 +9678,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([ws_order_number@1, ws_item_sk@0], 3), input_partitions=3 @@ -9936,38 +9718,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=3 - │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=2 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 28 ── tasks=2, partitions=3 │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 @@ -9979,17 +9759,17 @@ mod tests { │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(cs_quantity@2 AS Int64) - CASE WHEN __common_expr_7@0 IS NOT NULL THEN __common_expr_7@0 ELSE 0 END as sales_cnt, cs_ext_sales_price@3 - CASE WHEN __common_expr_8@1 IS NOT NULL THEN __common_expr_8@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(cs_quantity@2 AS Int64) - CASE WHEN __common_expr_7@0 IS NOT NULL THEN __common_expr_7@0 ELSE 0 END as sales_cnt, cs_ext_sales_price@3 - CASE WHEN __common_expr_8@1 IS NOT NULL THEN __common_expr_8@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(cr_return_quantity@0 AS Int64) as __common_expr_7, CAST(cr_return_amount@1 AS Decimal128(30, 15)) as __common_expr_8, cs_quantity@2 as cs_quantity, cs_ext_sales_price@3 as cs_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cr_return_quantity@11, cr_return_amount@12, cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ss_quantity@2 AS Int64) - CASE WHEN __common_expr_9@0 IS NOT NULL THEN __common_expr_9@0 ELSE 0 END as sales_cnt, ss_ext_sales_price@3 - CASE WHEN __common_expr_10@1 IS NOT NULL THEN __common_expr_10@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ss_quantity@2 AS Int64) - CASE WHEN __common_expr_9@0 IS NOT NULL THEN __common_expr_9@0 ELSE 0 END as sales_cnt, ss_ext_sales_price@3 - CASE WHEN __common_expr_10@1 IS NOT NULL THEN __common_expr_10@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(sr_return_quantity@0 AS Int64) as __common_expr_9, CAST(sr_return_amt@1 AS Decimal128(30, 15)) as __common_expr_10, ss_quantity@2 as ss_quantity, ss_ext_sales_price@3 as ss_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[sr_return_quantity@11, sr_return_amt@12, ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ws_quantity@2 AS Int64) - CASE WHEN __common_expr_11@0 IS NOT NULL THEN __common_expr_11@0 ELSE 0 END as sales_cnt, ws_ext_sales_price@3 - CASE WHEN __common_expr_12@1 IS NOT NULL THEN __common_expr_12@1 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[d_year@8 as d_year, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, CAST(ws_quantity@2 AS Int64) - CASE WHEN __common_expr_11@0 IS NOT NULL THEN __common_expr_11@0 ELSE 0 END as sales_cnt, ws_ext_sales_price@3 - CASE WHEN __common_expr_12@1 IS NOT NULL THEN __common_expr_12@1 ELSE 0.000000000000000 END as sales_amt] │ ProjectionExec: expr=[CAST(wr_return_quantity@0 AS Int64) as __common_expr_11, CAST(wr_return_amt@1 AS Decimal128(30, 15)) as __common_expr_12, ws_quantity@2 as ws_quantity, ws_ext_sales_price@3 as ws_ext_sales_price, i_brand_id@4 as i_brand_id, i_class_id@5 as i_class_id, i_category_id@6 as i_category_id, i_manufact_id@7 as i_manufact_id, d_year@8 as d_year] │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[wr_return_quantity@11, wr_return_amt@12, ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8] │ [Stage 25] => NetworkShuffleExec: output_partitions=3, input_tasks=4 @@ -10004,38 +9784,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 18 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 6), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 21 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([ss_ticket_number@1, ss_item_sk@0], 3), input_partitions=3 @@ -10046,38 +9824,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 20 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 22 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 25 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([ws_order_number@1, ws_item_sk@0], 3), input_partitions=3 @@ -10088,38 +9864,36 @@ mod tests { │ CoalescePartitionsExec │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 23 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── ┌───── Stage 24 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── ┌───── Stage 26 ── tasks=4, partitions=3 - │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=2 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -10127,16 +9901,17 @@ mod tests { #[tokio::test] async fn test_tpcds_76() -> Result<()> { let display = test_tpcds_query("q76").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category, count(Int64(1))@5 as sales_cnt, sum(foo.ext_sales_price)@6 as sales_amt] - │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([channel@0, col_name@1, d_year@2, d_qoy@3, i_category@4], 6), input_partitions=3 @@ -10145,82 +9920,82 @@ mod tests { │ ProjectionExec: expr=[store as channel, ss_store_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@2 as i_category, ss_ext_sales_price@3 as ext_sales_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, i_category@5, ss_ext_sales_price@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@2, ss_ext_sales_price@4, i_category@1] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=4 │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@7 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@7 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@7 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@7 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] │ ProjectionExec: expr=[web as channel, ws_ship_customer_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@2 as i_category, ws_ext_sales_price@3 as ext_sales_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, i_category@5, ws_ext_sales_price@4] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@2, ws_ext_sales_price@4, i_category@1] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@8 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@8 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] │ ProjectionExec: expr=[catalog as channel, cs_ship_addr_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@2 as i_category, cs_ext_sales_price@3 as ext_sales_price] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, i_category@5, cs_ext_sales_price@4] │ CoalescePartitionsExec - │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=4 + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[cs_sold_date_sk@2, cs_ext_sales_price@4, i_category@1] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=4 │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@10 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@10 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=16 - │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + ┌───── Stage 1 ── tasks=4, partitions=24 + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=16 - │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + ┌───── Stage 2 ── tasks=4, partitions=24 + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=8 - │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + ┌───── Stage 3 ── tasks=4, partitions=12 + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=8 - │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + ┌───── Stage 4 ── tasks=4, partitions=12 + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=8 - │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + ┌───── Stage 5 ── tasks=4, partitions=12 + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=8 - │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + ┌───── Stage 6 ── tasks=4, partitions=12 + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── "); Ok(()) @@ -10240,7 +10015,7 @@ mod tests { │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] │ InterleaveExec - │ ProjectionExec: expr=[store channel as channel, s_store_sk@2 as id, sales@3 as sales, CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE Some(0),22,2 END as returns_, profit@4 - CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE Some(0),22,2 END as profit] + │ ProjectionExec: expr=[store channel as channel, s_store_sk@2 as id, sales@3 as sales, CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE 0.00 END as returns_, profit@4 - CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE 0.00 END as profit] │ ProjectionExec: expr=[CAST(returns_@0 AS Decimal128(22, 2)) as __common_expr_1, CAST(profit_loss@1 AS Decimal128(22, 2)) as __common_expr_2, s_store_sk@2 as s_store_sk, sales@3 as sales, profit@4 as profit] │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(s_store_sk@0, s_store_sk@0)], projection=[returns_@1, profit_loss@2, s_store_sk@3, sales@4, profit@5] │ CoalescePartitionsExec @@ -10255,7 +10030,7 @@ mod tests { │ ProjectionExec: expr=[cs_call_center_sk@0 as cs_call_center_sk, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(catalog_sales.cs_net_profit)@2 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ ProjectionExec: expr=[web channel as channel, wp_web_page_sk@2 as id, sales@3 as sales, CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE Some(0),22,2 END as returns_, profit@4 - CASE WHEN __common_expr_4@1 IS NOT NULL THEN __common_expr_4@1 ELSE Some(0),22,2 END as profit] + │ ProjectionExec: expr=[web channel as channel, wp_web_page_sk@2 as id, sales@3 as sales, CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE 0.00 END as returns_, profit@4 - CASE WHEN __common_expr_4@1 IS NOT NULL THEN __common_expr_4@1 ELSE 0.00 END as profit] │ ProjectionExec: expr=[CAST(returns_@0 AS Decimal128(22, 2)) as __common_expr_3, CAST(profit_loss@1 AS Decimal128(22, 2)) as __common_expr_4, wp_web_page_sk@2 as wp_web_page_sk, sales@3 as sales, profit@4 as profit] │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(wp_web_page_sk@0, wp_web_page_sk@0)], projection=[returns_@1, profit_loss@2, wp_web_page_sk@3, sales@4, profit@5] │ CoalescePartitionsExec @@ -10280,10 +10055,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10296,12 +10071,11 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([s_store_sk@0], 6), input_partitions=3 @@ -10313,10 +10087,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10329,12 +10103,11 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 @@ -10349,20 +10122,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cs_call_center_sk@0], 6), input_partitions=3 @@ -10371,20 +10143,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=2, partitions=12 │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 @@ -10401,12 +10172,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, wr_returned_date_sk@0)], projection=[wr_web_page_sk@2, wr_return_amt@3, wr_net_loss@4] │ CoalescePartitionsExec │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10419,12 +10189,11 @@ mod tests { ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 6), input_partitions=3 @@ -10436,10 +10205,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10452,12 +10221,11 @@ mod tests { ┌───── Stage 18 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -10465,14 +10233,14 @@ mod tests { #[tokio::test] async fn test_tpcds_78() -> Result<()> { let display = test_tpcds_query("q78").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec - │ SortPreservingMergeExec: [ss_sold_year@0 ASC NULLS LAST, ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, store_sales_price@6 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST], fetch=100 + │ SortPreservingMergeExec: [ss_sold_year@0 ASC NULLS LAST, ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], fetch=100 │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, store_sales_price@6 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[ss_sold_year@5 as ss_sold_year, ss_item_sk@6 as ss_item_sk, ss_customer_sk@7 as ss_customer_sk, round(CAST(ss_qty@8 AS Float64) / CAST(__common_expr_1@0 AS Float64), 2) as ratio, ss_qty@8 as store_qty, ss_wc@9 as store_wholesale_cost, ss_sp@10 as store_sales_price, __common_expr_1@0 as other_chan_qty, CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE Some(0),22,2 END + CASE WHEN __common_expr_3@2 IS NOT NULL THEN __common_expr_3@2 ELSE Some(0),22,2 END as other_chan_wholesale_cost, CASE WHEN __common_expr_4@3 IS NOT NULL THEN __common_expr_4@3 ELSE Some(0),22,2 END + CASE WHEN __common_expr_5@4 IS NOT NULL THEN __common_expr_5@4 ELSE Some(0),22,2 END as other_chan_sales_price] + │ SortExec: TopK(fetch=100), expr=[ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_sold_year@5 as ss_sold_year, ss_item_sk@6 as ss_item_sk, ss_customer_sk@7 as ss_customer_sk, round(CAST(ss_qty@8 AS Float64) / CAST(__common_expr_1@0 AS Float64), 2) as ratio, ss_qty@8 as store_qty, ss_wc@9 as store_wholesale_cost, ss_sp@10 as store_sales_price, __common_expr_1@0 as other_chan_qty, CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE 0.00 END + CASE WHEN __common_expr_3@2 IS NOT NULL THEN __common_expr_3@2 ELSE 0.00 END as other_chan_wholesale_cost, CASE WHEN __common_expr_4@3 IS NOT NULL THEN __common_expr_4@3 ELSE 0.00 END + CASE WHEN __common_expr_5@4 IS NOT NULL THEN __common_expr_5@4 ELSE 0.00 END as other_chan_sales_price] │ ProjectionExec: expr=[CASE WHEN ws_qty@6 IS NOT NULL THEN ws_qty@6 ELSE 0 END + CASE WHEN cs_qty@9 IS NOT NULL THEN cs_qty@9 ELSE 0 END as __common_expr_1, CAST(ws_wc@7 AS Decimal128(22, 2)) as __common_expr_2, CAST(cs_wc@10 AS Decimal128(22, 2)) as __common_expr_3, CAST(ws_sp@8 AS Decimal128(22, 2)) as __common_expr_4, CAST(cs_sp@11 AS Decimal128(22, 2)) as __common_expr_5, ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_qty@3 as ss_qty, ss_wc@4 as ss_wc, ss_sp@5 as ss_sp] │ FilterExec: CASE WHEN ws_qty@6 IS NOT NULL THEN ws_qty@6 ELSE 0 END > 0 OR CASE WHEN cs_qty@9 IS NOT NULL THEN cs_qty@9 ELSE 0 END > 0 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cs_sold_year@0, ss_sold_year@0), (cs_item_sk@1, ss_item_sk@1), (cs_customer_sk@2, ss_customer_sk@2)], projection=[ss_sold_year@6, ss_item_sk@7, ss_customer_sk@8, ss_qty@9, ss_wc@10, ss_sp@11, ws_qty@12, ws_wc@13, ws_sp@14, cs_qty@3, cs_wc@4, cs_sp@5] @@ -10501,12 +10269,11 @@ mod tests { ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 12), input_partitions=3 @@ -10519,10 +10286,10 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_year@0, ws_item_sk@1, ws_bill_customer_sk@2], 6), input_partitions=3 @@ -10533,30 +10300,29 @@ mod tests { │ FilterExec: wr_order_number@6 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_bill_customer_sk@2, ws_quantity@3, ws_wholesale_cost@4, ws_sales_price@5] │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(wr_order_number@1, ws_order_number@3), (wr_item_sk@0, ws_item_sk@1)], projection=[ws_sold_date_sk@2, ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8, wr_order_number@1] │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([d_year@0, ss_item_sk@1, ss_customer_sk@2], 6), input_partitions=3 @@ -10572,12 +10338,11 @@ mod tests { ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 12), input_partitions=3 @@ -10590,10 +10355,10 @@ mod tests { ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -10601,7 +10366,7 @@ mod tests { #[tokio::test] async fn test_tpcds_79() -> Result<()> { let display = test_tpcds_query("q79").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], fetch=100 │ [Stage 6] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 @@ -10611,18 +10376,18 @@ mod tests { │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, substr(s_city@2, 1, 30) as substr(ms.s_city,Int64(1),Int64(30)), ss_ticket_number@3 as ss_ticket_number, amt@4 as amt, profit@5 as profit] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@1)], projection=[c_last_name@2, c_first_name@1, s_city@5, ss_ticket_number@3, amt@6, profit@7] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, s_city@3 as s_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, s_city@3 as s_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, s_city@3], 12), input_partitions=3 @@ -10637,10 +10402,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -10665,12 +10430,11 @@ mod tests { ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_dow@2 = 1 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] └────────────────────────────────────────────────── "); Ok(()) @@ -10678,7 +10442,7 @@ mod tests { #[tokio::test] async fn test_tpcds_80() -> Result<()> { let display = test_tpcds_query("q80").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] @@ -10691,18 +10455,18 @@ mod tests { │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(coalesce(store_returns.sr_return_amt,Int64(0)))@2 as returns_, sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Some(0),22,2 END) as sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE Some(0),22,2 END) as sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0.00 END) as sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0.00 END) as sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Some(0),22,2 END) as sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE Some(0),22,2 END) as sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE 0.00 END) as sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE 0.00 END) as sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE Some(0),22,2 END) as sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE Some(0),22,2 END) as sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0.00 END) as sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0.00 END) as sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([s_store_id@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE Some(0),22,2 END) as sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE Some(0),22,2 END) as sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(CASE WHEN __common_expr_1 IS NOT NULL THEN __common_expr_1 ELSE 0.00 END) as sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - CASE WHEN __common_expr_2 IS NOT NULL THEN __common_expr_2 ELSE 0.00 END) as sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] │ ProjectionExec: expr=[CAST(sr_return_amt@0 AS Decimal128(22, 2)) as __common_expr_1, CAST(sr_net_loss@1 AS Decimal128(22, 2)) as __common_expr_2, ss_ext_sales_price@2 as ss_ext_sales_price, ss_net_profit@3 as ss_net_profit, s_store_id@4 as s_store_id] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_promo_sk@0, ss_promo_sk@0)], projection=[sr_return_amt@4, sr_net_loss@5, ss_ext_sales_price@2, ss_net_profit@3, s_store_id@6] │ CoalescePartitionsExec @@ -10732,13 +10496,12 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@1 > Some(5000),7,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] + │ FilterExec: i_current_price@1 > 50.00, projection=[i_item_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10751,12 +10514,11 @@ mod tests { ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 12), input_partitions=3 @@ -10769,14 +10531,14 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE Some(0),22,2 END) as sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE Some(0),22,2 END) as sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(CASE WHEN __common_expr_3 IS NOT NULL THEN __common_expr_3 ELSE 0.00 END) as sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - CASE WHEN __common_expr_4 IS NOT NULL THEN __common_expr_4 ELSE 0.00 END) as sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] │ ProjectionExec: expr=[CAST(cr_return_amount@0 AS Decimal128(22, 2)) as __common_expr_3, CAST(cr_net_loss@1 AS Decimal128(22, 2)) as __common_expr_4, cs_ext_sales_price@2 as cs_ext_sales_price, cs_net_profit@3 as cs_net_profit, cp_catalog_page_id@4 as cp_catalog_page_id] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_promo_sk@0, cs_promo_sk@0)], projection=[cr_return_amount@4, cr_net_loss@5, cs_ext_sales_price@2, cs_net_profit@3, cp_catalog_page_id@6] │ CoalescePartitionsExec @@ -10786,7 +10548,7 @@ mod tests { │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cp_catalog_page_sk@0, cs_catalog_page_sk@0)], projection=[cs_item_sk@3, cs_promo_sk@4, cs_ext_sales_price@5, cs_net_profit@6, cr_return_amount@7, cr_net_loss@8, cp_catalog_page_id@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_catalog_page_sk@2, cs_item_sk@3, cs_promo_sk@4, cs_ext_sales_price@5, cs_net_profit@6, cr_return_amount@7, cr_net_loss@8] │ CoalescePartitionsExec │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -10806,31 +10568,29 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@1 > Some(5000),7,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] + │ FilterExec: i_current_price@1 > 50.00, projection=[i_item_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet:..]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 12), input_partitions=3 @@ -10843,14 +10603,14 @@ mod tests { ┌───── Stage 13 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 21 ── tasks=4, partitions=3 │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE Some(0),22,2 END) as sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE Some(0),22,2 END) as sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(CASE WHEN __common_expr_5 IS NOT NULL THEN __common_expr_5 ELSE 0.00 END) as sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - CASE WHEN __common_expr_6 IS NOT NULL THEN __common_expr_6 ELSE 0.00 END) as sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] │ ProjectionExec: expr=[CAST(wr_return_amt@0 AS Decimal128(22, 2)) as __common_expr_5, CAST(wr_net_loss@1 AS Decimal128(22, 2)) as __common_expr_6, ws_ext_sales_price@2 as ws_ext_sales_price, ws_net_profit@3 as ws_net_profit, web_site_id@4 as web_site_id] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_promo_sk@0, ws_promo_sk@0)], projection=[wr_return_amt@4, wr_net_loss@5, ws_ext_sales_price@2, ws_net_profit@3, web_site_id@6] │ CoalescePartitionsExec @@ -10880,13 +10640,12 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 16 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@1 > Some(5000),7,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),7,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),7,2, required_guarantees=[] + │ FilterExec: i_current_price@1 > 50.00, projection=[i_item_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > 50.00, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > 50.00, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -10899,28 +10658,27 @@ mod tests { ┌───── Stage 18 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 12), input_partitions=2 + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 20 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -10928,44 +10686,39 @@ mod tests { #[tokio::test] async fn test_tpcds_81() -> Result<()> { let display = test_tpcds_query("q81").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_state@10 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], fetch=100 - │ [Stage 11] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 12] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=3 + ┌───── Stage 12 ── tasks=4, partitions=3 │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_state@0, ctr_state@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@13, c_salutation@14, c_first_name@15, c_last_name@16, ca_street_number@2, ca_street_name@3, ca_street_type@4, ca_suite_number@5, ca_city@6, ca_county@7, ca_state@8, ca_zip@9, ca_country@10, ca_gmt_offset@11, ca_location_type@12, ctr_total_return@1] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_state@0 as ctr_state] - │ AggregateExec: mode=FinalPartitioned, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ctr_state@0, ctr_state@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@13, c_salutation@14, c_first_name@15, c_last_name@16, ca_street_number@2, ca_street_name@3, ca_street_type@4, ca_suite_number@5, ca_city@6, ca_county@7, ca_state@8, ca_zip@9, ca_country@10, ca_gmt_offset@11, ca_location_type@12, ctr_total_return@1] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=1 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 6 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ctr_state@0], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ctr_state@12, ctr_total_return@13, ca_street_number@1, ca_street_name@2, ca_street_type@3, ca_suite_number@4, ca_city@5, ca_county@6, ca_state@7, ca_zip@8, ca_country@9, ca_gmt_offset@10, ca_location_type@11, c_customer_id@14, c_salutation@16, c_first_name@17, c_last_name@18] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, c_customer_sk@0)], projection=[ctr_state@1, ctr_total_return@2, c_customer_id@4, c_current_addr_sk@5, c_salutation@6, c_first_name@7, c_last_name@8] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_state@7 = GA - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=2, partitions=24 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -10979,12 +10732,11 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, ca_address_sk@0)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -10992,93 +10744,91 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=2, partitions=12 - │ RepartitionExec: partitioning=Hash([ctr_state@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] - │ ProjectionExec: expr=[ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] - │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=6 - │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, ca_address_sk@0)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 11 ── tasks=1, partitions=12 + │ RepartitionExec: partitioning=Hash([ctr_state@1], 12), input_partitions=3 + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_state@0 as ctr_state] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── tasks=2, partitions=3 + │ RepartitionExec: partitioning=Hash([ctr_state@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + ┌───── Stage 9 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, ca_address_sk@0)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=48 + ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_82() -> Result<()> { let display = test_tpcds_query("q82").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=2, partitions=3 + ┌───── Stage 6 ── tasks=2, partitions=3 │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=6 + ┌───── Stage 5 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@4)], projection=[i_item_sk@1, i_item_id@2, i_item_desc@3, i_current_price@4] + ┌───── Stage 3 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, inv_date_sk@4)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3, inv_date_sk@4] @@ -11086,31 +10836,37 @@ mod tests { │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-05-25 AND d_date@1 <= 2000-07-24, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: i_current_price@3 >= Some(6200),7,2 AND i_current_price@3 <= Some(9200),7,2 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6200),7,2 AND i_current_price@5 <= Some(9200),7,2 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6200),7,2 AND i_current_price@5 <= Some(9200),7,2 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6200),7,2 AND i_current_price@5 <= Some(9200),7,2 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6200),7,2 AND i_current_price@5 <= Some(9200),7,2 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),7,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),7,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + │ FilterExec: i_current_price@3 >= 62.00 AND i_current_price@3 <= 92.00 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 62.00 AND i_current_price@5 <= 92.00 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 62.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 92.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 62.00 AND i_current_price@5 <= 92.00 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 62.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 92.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 62.00 AND i_current_price@5 <= 92.00 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 62.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 92.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= 62.00 AND i_current_price@5 <= 92.00 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= 62.00 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= 92.00 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── "); Ok(()) } @@ -11155,54 +10911,50 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_returned_date_sk@0, d_date_sk@0)], projection=[wr_return_quantity@1, i_item_id@2, d_date@4] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, wr_item_sk@1)], projection=[wr_returned_date_sk@2, wr_return_quantity@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([d_date@0], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] └────────────────────────────────────────────────── ┌───── Stage 12 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -11212,53 +10964,51 @@ mod tests { │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_return_quantity@3, i_item_id@4, d_date@1] │ CoalescePartitionsExec - │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cr_item_sk@1)], projection=[cr_returned_date_sk@2, cr_return_quantity@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] └────────────────────────────────────────────────── - ┌───── Stage 10 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 10 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 11 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 11 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 @@ -11268,53 +11018,51 @@ mod tests { │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, sr_returned_date_sk@0)], projection=[sr_return_quantity@3, i_item_id@4, d_date@1] │ CoalescePartitionsExec - │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, sr_item_sk@1)], projection=[sr_returned_date_sk@2, sr_return_quantity@4, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] │ CoalescePartitionsExec │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] └────────────────────────────────────────────────── - ┌───── Stage 15 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 15 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 16 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 16 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -11334,10 +11082,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11351,10 +11099,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11369,32 +11117,30 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet:..]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[c_customer_id@1, c_current_cdemo_sk@2, c_current_hdemo_sk@3, c_first_name@5, c_last_name@6] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_city@1 = Edgewood, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] └────────────────────────────────────────────────── "); Ok(()) @@ -11402,7 +11148,7 @@ mod tests { #[tokio::test] async fn test_tpcds_85() -> Result<()> { let display = test_tpcds_query("q85").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [substr(reason.r_reason_desc,Int64(1),Int64(20))@0 ASC NULLS LAST, avg1@1 ASC NULLS LAST, avg2@2 ASC NULLS LAST, avg(web_returns.wr_fee)@3 ASC NULLS LAST], fetch=100 │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 @@ -11422,7 +11168,7 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_quantity@2, wr_reason_sk@3, wr_fee@4, wr_refunded_cash@5] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, wr_refunded_addr_sk@3)], filter=(ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ) AND ws_net_profit@0 >= Some(10000),7,2 AND ws_net_profit@0 <= Some(20000),7,2 OR (ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY) AND ws_net_profit@0 >= Some(15000),7,2 AND ws_net_profit@0 <= Some(30000),7,2 OR (ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ws_net_profit@0 >= Some(5000),7,2 AND ws_net_profit@0 <= Some(25000),7,2, projection=[ws_sold_date_sk@2, ws_quantity@3, wr_reason_sk@6, wr_fee@7, wr_refunded_cash@8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, wr_refunded_addr_sk@3)], filter=(ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ) AND ws_net_profit@0 >= 100.00 AND ws_net_profit@0 <= 200.00 OR (ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY) AND ws_net_profit@0 >= 150.00 AND ws_net_profit@0 <= 300.00 OR (ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ws_net_profit@0 >= 50.00 AND ws_net_profit@0 <= 250.00, projection=[ws_sold_date_sk@2, ws_quantity@3, wr_reason_sk@6, wr_fee@7, wr_refunded_cash@8] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_returning_cdemo_sk@4, cd_demo_sk@0), (cd_marital_status@8, cd_marital_status@1), (cd_education_status@9, cd_education_status@2)], projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@2, wr_refunded_addr_sk@3, wr_reason_sk@5, wr_fee@6, wr_refunded_cash@7] @@ -11440,26 +11186,24 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_country@2 = United States AND ca_state@1 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ OR ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY OR ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR), projection=[ca_address_sk@0, ca_state@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND (ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5) AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IN AND IN <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NJ AND NJ <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= WI AND WI <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= CT AND CT <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= LA AND LA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= IA AND IA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= AR AND AR <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([wr_returning_cdemo_sk@4, cd_marital_status@8, cd_education_status@9], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_refunded_cdemo_sk@4, cd_demo_sk@0)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ws_sales_price@0 >= Some(10000),7,2 AND ws_sales_price@0 <= Some(15000),7,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ws_sales_price@0 >= Some(5000),7,2 AND ws_sales_price@0 <= Some(10000),7,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ws_sales_price@0 >= Some(15000),7,2 AND ws_sales_price@0 <= Some(20000),7,2, projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@3, wr_refunded_addr_sk@5, wr_returning_cdemo_sk@6, wr_reason_sk@7, wr_fee@8, wr_refunded_cash@9, cd_marital_status@11, cd_education_status@12] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_refunded_cdemo_sk@4, cd_demo_sk@0)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ws_sales_price@0 >= 100.00 AND ws_sales_price@0 <= 150.00 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ws_sales_price@0 >= 50.00 AND ws_sales_price@0 <= 100.00 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ws_sales_price@0 >= 150.00 AND ws_sales_price@0 <= 200.00, projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@3, wr_refunded_addr_sk@5, wr_returning_cdemo_sk@6, wr_reason_sk@7, wr_fee@8, wr_refunded_cash@9, cd_marital_status@11, cd_education_status@12] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -11481,38 +11225,38 @@ mod tests { │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet:..]]}, projection=[wp_web_page_sk], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@5], 12), input_partitions=2 + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@5], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@3], 12), input_partitions=3 - │ FilterExec: (ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),7,2 AND ws_sales_price@5 <= Some(15000),7,2 OR ws_sales_price@5 >= Some(5000),7,2 AND ws_sales_price@5 <= Some(10000),7,2 OR ws_sales_price@5 >= Some(15000),7,2 AND ws_sales_price@5 <= Some(20000),7,2) + │ FilterExec: (ws_net_profit@6 >= 100.00 AND ws_net_profit@6 <= 200.00 OR ws_net_profit@6 >= 150.00 AND ws_net_profit@6 <= 300.00 OR ws_net_profit@6 >= 50.00 AND ws_net_profit@6 <= 250.00) AND (ws_sales_price@5 >= 100.00 AND ws_sales_price@5 <= 150.00 OR ws_sales_price@5 >= 50.00 AND ws_sales_price@5 <= 100.00 OR ws_sales_price@5 >= 150.00 AND ws_sales_price@5 <= 200.00) │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= Some(10000),7,2 AND ws_net_profit@33 <= Some(20000),7,2 OR ws_net_profit@33 >= Some(15000),7,2 AND ws_net_profit@33 <= Some(30000),7,2 OR ws_net_profit@33 >= Some(5000),7,2 AND ws_net_profit@33 <= Some(25000),7,2) AND (ws_sales_price@21 >= Some(10000),7,2 AND ws_sales_price@21 <= Some(15000),7,2 OR ws_sales_price@21 >= Some(5000),7,2 AND ws_sales_price@21 <= Some(10000),7,2 OR ws_sales_price@21 >= Some(15000),7,2 AND ws_sales_price@21 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= Some(10000),7,2 AND ws_net_profit@33 <= Some(20000),7,2 OR ws_net_profit@33 >= Some(15000),7,2 AND ws_net_profit@33 <= Some(30000),7,2 OR ws_net_profit@33 >= Some(5000),7,2 AND ws_net_profit@33 <= Some(25000),7,2) AND (ws_sales_price@21 >= Some(10000),7,2 AND ws_sales_price@21 <= Some(15000),7,2 OR ws_sales_price@21 >= Some(5000),7,2 AND ws_sales_price@21 <= Some(10000),7,2 OR ws_sales_price@21 >= Some(15000),7,2 AND ws_sales_price@21 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= Some(10000),7,2 AND ws_net_profit@33 <= Some(20000),7,2 OR ws_net_profit@33 >= Some(15000),7,2 AND ws_net_profit@33 <= Some(30000),7,2 OR ws_net_profit@33 >= Some(5000),7,2 AND ws_net_profit@33 <= Some(25000),7,2) AND (ws_sales_price@21 >= Some(10000),7,2 AND ws_sales_price@21 <= Some(15000),7,2 OR ws_sales_price@21 >= Some(5000),7,2 AND ws_sales_price@21 <= Some(10000),7,2 OR ws_sales_price@21 >= Some(15000),7,2 AND ws_sales_price@21 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= Some(10000),7,2 AND ws_net_profit@33 <= Some(20000),7,2 OR ws_net_profit@33 >= Some(15000),7,2 AND ws_net_profit@33 <= Some(30000),7,2 OR ws_net_profit@33 >= Some(5000),7,2 AND ws_net_profit@33 <= Some(25000),7,2) AND (ws_sales_price@21 >= Some(10000),7,2 AND ws_sales_price@21 <= Some(15000),7,2 OR ws_sales_price@21 >= Some(5000),7,2 AND ws_sales_price@21 <= Some(10000),7,2 OR ws_sales_price@21 >= Some(15000),7,2 AND ws_sales_price@21 <= Some(20000),7,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),7,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),7,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),7,2), required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= 100.00 AND ws_net_profit@33 <= 200.00 OR ws_net_profit@33 >= 150.00 AND ws_net_profit@33 <= 300.00 OR ws_net_profit@33 >= 50.00 AND ws_net_profit@33 <= 250.00) AND (ws_sales_price@21 >= 100.00 AND ws_sales_price@21 <= 150.00 OR ws_sales_price@21 >= 50.00 AND ws_sales_price@21 <= 100.00 OR ws_sales_price@21 >= 150.00 AND ws_sales_price@21 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 100.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 200.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 150.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 300.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 50.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 250.00) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 100.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 150.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 50.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 100.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 150.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= 100.00 AND ws_net_profit@33 <= 200.00 OR ws_net_profit@33 >= 150.00 AND ws_net_profit@33 <= 300.00 OR ws_net_profit@33 >= 50.00 AND ws_net_profit@33 <= 250.00) AND (ws_sales_price@21 >= 100.00 AND ws_sales_price@21 <= 150.00 OR ws_sales_price@21 >= 50.00 AND ws_sales_price@21 <= 100.00 OR ws_sales_price@21 >= 150.00 AND ws_sales_price@21 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 100.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 200.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 150.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 300.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 50.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 250.00) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 100.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 150.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 50.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 100.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 150.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= 100.00 AND ws_net_profit@33 <= 200.00 OR ws_net_profit@33 >= 150.00 AND ws_net_profit@33 <= 300.00 OR ws_net_profit@33 >= 50.00 AND ws_net_profit@33 <= 250.00) AND (ws_sales_price@21 >= 100.00 AND ws_sales_price@21 <= 150.00 OR ws_sales_price@21 >= 50.00 AND ws_sales_price@21 <= 100.00 OR ws_sales_price@21 >= 150.00 AND ws_sales_price@21 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 100.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 200.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 150.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 300.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 50.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 250.00) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 100.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 150.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 50.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 100.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 150.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 200.00), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= 100.00 AND ws_net_profit@33 <= 200.00 OR ws_net_profit@33 >= 150.00 AND ws_net_profit@33 <= 300.00 OR ws_net_profit@33 >= 50.00 AND ws_net_profit@33 <= 250.00) AND (ws_sales_price@21 >= 100.00 AND ws_sales_price@21 <= 150.00 OR ws_sales_price@21 >= 50.00 AND ws_sales_price@21 <= 100.00 OR ws_sales_price@21 >= 150.00 AND ws_sales_price@21 <= 200.00) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 100.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 200.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 150.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 300.00 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= 50.00 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= 250.00) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 100.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 150.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 50.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 100.00 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= 150.00 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= 200.00), required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0], 12), input_partitions=3 │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([cd_demo_sk@0, cd_marital_status@1, cd_education_status@2], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -11568,7 +11312,7 @@ mod tests { #[tokio::test] async fn test_tpcds_87() -> Result<()> { let display = test_tpcds_query("q87").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] @@ -11580,112 +11324,107 @@ mod tests { │ HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], projection=[], NullsEqual: true │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true - │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ws_bill_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 1 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, cs_bill_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_customer_sk@0, ss_customer_sk@0)], projection=[c_last_name@2, c_first_name@1, d_date@4] │ CoalescePartitionsExec - │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, d_date@1] │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -11767,10 +11506,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11785,12 +11524,11 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 12 AND t_minute@2 < 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11814,10 +11552,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11832,12 +11570,11 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 11 AND t_minute@2 >= 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11861,10 +11598,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11879,12 +11616,11 @@ mod tests { ┌───── Stage 10 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 11 AND t_minute@2 < 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11908,10 +11644,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 13 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11926,12 +11662,11 @@ mod tests { ┌───── Stage 14 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 10 AND t_minute@2 >= 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] └────────────────────────────────────────────────── ┌───── Stage 15 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11955,10 +11690,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 17 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -11973,12 +11708,11 @@ mod tests { ┌───── Stage 18 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 10 AND t_minute@2 < 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] └────────────────────────────────────────────────── ┌───── Stage 19 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12002,10 +11736,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 21 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12020,12 +11754,11 @@ mod tests { ┌───── Stage 22 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 9 AND t_minute@2 >= 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] └────────────────────────────────────────────────── ┌───── Stage 23 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12049,10 +11782,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 25 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12067,12 +11800,11 @@ mod tests { ┌───── Stage 26 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 8 AND t_minute@2 >= 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] └────────────────────────────────────────────────── ┌───── Stage 27 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12096,10 +11828,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 31] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 29 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12114,12 +11846,11 @@ mod tests { ┌───── Stage 30 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 9 AND t_minute@2 < 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] └────────────────────────────────────────────────── ┌───── Stage 31 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12142,7 +11873,7 @@ mod tests { │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], fetch=100 │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, s_store_name@3 as s_store_name, s_company_name@4 as s_company_name, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 as avg_monthly_sales] - │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 != Some(0),21,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 END > Some(1000000000),32,10 + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 != 0.000000 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 END > 0.1000000000 │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(21, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST], preserve_partitioning=[true] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -12165,10 +11896,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -12181,22 +11912,20 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_year@1 = 1999, projection=[d_date_sk@0, d_moy@2] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: (i_category@3 = Books OR i_category@3 = Electronics OR i_category@3 = Sports) AND (i_class@2 = computers OR i_class@2 = stereo OR i_class@2 = football) OR (i_category@3 = Men OR i_category@3 = Jewelry OR i_category@3 = Women) AND (i_class@2 = shirts OR i_class@2 = birdal OR i_class@2 = dresses) - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] └────────────────────────────────────────────────── "#); Ok(()) @@ -12207,7 +11936,7 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec │ SortExec: TopK(fetch=100), expr=[am_pm_ratio@0 ASC NULLS LAST], preserve_partitioning=[false] - │ ProjectionExec: expr=[CASE WHEN pmc@1 = 0 THEN None,23,8 ELSE CAST(amc@0 AS Decimal128(15, 4)) / CAST(pmc@1 AS Decimal128(15, 4)) END as am_pm_ratio] + │ ProjectionExec: expr=[CASE WHEN pmc@1 = 0 THEN NULL ELSE CAST(amc@0 AS Decimal128(15, 4)) / CAST(pmc@1 AS Decimal128(15, 4)) END as am_pm_ratio] │ CrossJoinExec │ CoalescePartitionsExec │ BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 @@ -12232,10 +11961,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12250,12 +11979,11 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 >= 8 AND t_hour@1 <= 9, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12279,10 +12007,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12297,12 +12025,11 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 >= 19 AND t_hour@1 <= 20, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12342,10 +12069,10 @@ mod tests { │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Unknown OR cd_marital_status@1 = W AND cd_education_status@2 = Advanced Degree │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=(cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12362,25 +12089,23 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@6, ca_address_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, c_current_cdemo_sk@4, c_current_hdemo_sk@5] │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: ca_gmt_offset@1 = Some(-700),5,2, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-700),5,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),5,2 AND Some(-700),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),5,2)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-700),5,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),5,2 AND Some(-700),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),5,2)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-700),5,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),5,2 AND Some(-700),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),5,2)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-700),5,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),5,2 AND Some(-700),5,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),5,2)] + │ FilterExec: ca_gmt_offset@1 = -7.00, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -7.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -7.00 AND -7.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-7.00)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -7.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -7.00 AND -7.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-7.00)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -7.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -7.00 AND -7.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-7.00)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = -7.00 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= -7.00 AND -7.00 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (-7.00)] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_customer_sk@3, c_customer_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@4, c_current_cdemo_sk@6, c_current_hdemo_sk@7, c_current_addr_sk@8] │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet:..]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12391,20 +12116,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 @@ -12420,25 +12144,22 @@ mod tests { #[tokio::test] async fn test_tpcds_92() -> Result<()> { let display = test_tpcds_query("q92").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ ProjectionExec: expr=[sum(web_sales.ws_ext_discount_amt)@0 as Excess Discount Amount] │ GlobalLimitExec: skip=0, fetch=100 │ AggregateExec: mode=Final, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=3 + ┌───── Stage 7 ── tasks=4, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@1, ws_item_sk@1)], filter=CAST(ws_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(web_sales.ws_ext_discount_amt)@1, projection=[ws_ext_discount_amt@0] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(web_sales.ws_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(web_sales.ws_ext_discount_amt), ws_item_sk@0 as ws_item_sk] - │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(i_item_sk@1, ws_item_sk@1)], filter=CAST(ws_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(web_sales.ws_ext_discount_amt)@1, projection=[ws_ext_discount_amt@0] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 3 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([i_item_sk@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_ext_discount_amt@2, i_item_sk@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -12446,69 +12167,73 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_manufact_id@1 = 350, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350, pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_ext_discount_amt@3] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 6 ── tasks=2, partitions=12 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 12), input_partitions=3 + │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(web_sales.ws_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(web_sales.ws_ext_discount_amt), ws_item_sk@0 as ws_item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + ┌───── Stage 5 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_sold_date_sk@0)], projection=[ws_item_sk@2, ws_ext_discount_amt@3] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── + ┌───── Stage 4 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] async fn test_tpcds_93() -> Result<()> { let display = test_tpcds_query("q93").await?; - assert_snapshot!(display, @r" + assert_snapshot!(display, @" ┌───── DistributedExec │ SortPreservingMergeExec: [sumsales@1 ASC, ss_customer_sk@0 ASC], fetch=100 │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=2, partitions=3 - │ SortExec: TopK(fetch=100), expr=[sumsales@1 ASC, ss_customer_sk@0 ASC], preserve_partitioning=[true] + │ LocalLimitExec: fetch=100 │ ProjectionExec: expr=[ss_customer_sk@0 as ss_customer_sk, sum(t.act_sales)@1 as sumsales] - │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ SortExec: TopK(fetch=100), expr=[sum(t.act_sales)@1 ASC, ss_customer_sk@0 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 6), input_partitions=3 @@ -12591,10 +12316,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12609,39 +12334,36 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([wr_order_number@0], 12), input_partitions=3 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -12665,117 +12387,115 @@ mod tests { ┌───── Stage 13 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[ws_order_number@0 as alias1], aggr=[sum(ws1.ws_ext_ship_cost) as alias2, sum(ws1.ws_net_profit) as alias3] - │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_order_number@0, wr_order_number@0)] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(wr_order_number@0, ws_order_number@0)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_order_number@0, ws_order_number@0)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 8 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_order_number@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_order_number@0, ws_order_number@0)] + ┌───── Stage 5 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(wr_order_number@0, ws_order_number@0)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([ws_order_number@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(web_site_sk@0, ws_web_site_sk@0)], projection=[ws_order_number@2, ws_ext_ship_cost@3, ws_net_profit@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@2, ws_order_number@3, ws_ext_ship_cost@4, ws_net_profit@5] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@2, ws_web_site_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + ┌───── Stage 1 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([wr_order_number@0], 12), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=12 + ┌───── Stage 4 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@0], 12), input_partitions=3 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 + ┌───── Stage 2 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 + ┌───── Stage 3 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 12 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([wr_order_number@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_order_number@0, ws_order_number@0)], projection=[wr_order_number@0] + ┌───── Stage 9 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ws_order_number@0], 12), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(web_site_sk@0, ws_web_site_sk@0)], projection=[ws_order_number@2, ws_ext_ship_cost@3, ws_net_profit@4] │ CoalescePartitionsExec - │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@2, ws_order_number@3, ws_ext_ship_cost@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@2, ws_web_site_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 9 ── tasks=4, partitions=32 - │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet:..]]}, projection=[wr_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + ┌───── Stage 6 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet:..]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet:..]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── tasks=4, partitions=48 + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] └────────────────────────────────────────────────── + ┌───── Stage 12 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ws_order_number@0], 12), input_partitions=3 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── ┌───── Stage 10 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ws_order_number@1], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -12803,10 +12523,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12821,12 +12541,11 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: t_hour@1 = 20 AND t_minute@2 >= 30, projection=[t_time_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet:..]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -12866,46 +12585,44 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0, cs_item_sk@1], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cs_bill_customer_sk@0 as cs_bill_customer_sk, cs_item_sk@1 as cs_item_sk], aggr=[] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@2, cs_item_sk@3] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@1, cs_item_sk@2] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([ss_customer_sk@0, ss_item_sk@1], 6), input_partitions=3 │ AggregateExec: mode=Partial, gby=[ss_customer_sk@1 as ss_customer_sk, ss_item_sk@0 as ss_item_sk], aggr=[] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date_sk@0, ss_sold_date_sk@0)], projection=[ss_item_sk@1, ss_customer_sk@2] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -12937,30 +12654,28 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet:..]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] └────────────────────────────────────────────────── "#); Ok(()) @@ -12996,20 +12711,19 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t1: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t2: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] - │ t3: DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet:..]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=32 │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index fd534a3a7..70fa5e54a 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -20,27 +20,27 @@ mod tests { #[tokio::test] async fn test_tpch_1() -> Result<(), Box> { let plan = test_tpch_query("q1").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] - │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + │ ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] + │ SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * Some(1),20,0 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] - │ ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] + │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + │ ProjectionExec: expr=[l_extendedprice@0 * (1 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_extendedprice@1, l_discount@2, l_quantity@0, l_tax@3, l_returnflag@4, l_linestatus@5] │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -56,7 +56,7 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 11 ── tasks=4, partitions=3 │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6] + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── @@ -75,10 +75,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -92,18 +92,18 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -133,10 +133,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -150,18 +150,18 @@ mod tests { ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -170,29 +170,29 @@ mod tests { #[tokio::test] async fn test_tpch_3() -> Result<(), Box> { let plan = test_tpch_query("q3").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST] │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=3, partitions=3 - │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] + │ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 9), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -201,10 +201,10 @@ mod tests { │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: o_orderdate@2 < 1995-03-15 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -228,8 +228,8 @@ mod tests { │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=2, partitions=3 - │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── @@ -247,16 +247,16 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([l_orderkey@0], 9), input_partitions=3 │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -265,20 +265,20 @@ mod tests { #[tokio::test] async fn test_tpch_5() -> Result<(), Box> { let plan = test_tpch_query("q5").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [revenue@1 DESC] │ [Stage 7] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=3, partitions=3 - │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] + │ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([n_name@0], 9), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -292,10 +292,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -309,18 +309,18 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -329,10 +329,10 @@ mod tests { │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -358,12 +358,12 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] - │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] + │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= 0.05 AND l_discount@2 <= 0.07 AND l_quantity@0 < 24.00, projection=[l_extendedprice@1, l_discount@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= Some(5),15,2 AND l_discount@6 <= Some(7),15,2 AND l_quantity@4 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= Some(5),15,2 AND l_discount@6 <= Some(7),15,2 AND l_quantity@4 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= Some(5),15,2 AND l_discount@6 <= Some(7),15,2 AND l_quantity@4 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= Some(5),15,2 AND l_discount@6 <= Some(7),15,2 AND l_quantity@4 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= 0.05 AND l_discount@6 <= 0.07 AND l_quantity@4 < 24.00, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= 0.05 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= 0.07 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < 24.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= 0.05 AND l_discount@6 <= 0.07 AND l_quantity@4 < 24.00, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= 0.05 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= 0.07 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < 24.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= 0.05 AND l_discount@6 <= 0.07 AND l_quantity@4 < 24.00, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= 0.05 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= 0.07 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < 24.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= 0.05 AND l_discount@6 <= 0.07 AND l_quantity@4 < 24.00, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= 0.05 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= 0.07 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < 24.00, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -372,21 +372,21 @@ mod tests { #[tokio::test] async fn test_tpch_7() -> Result<(), Box> { let plan = test_tpch_query("q7").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] │ [Stage 7] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=3, partitions=3 - │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 9), input_partitions=3 │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - │ ProjectionExec: expr=[n_name@0 as supp_nation, n_name@1 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@3 * (Some(1),20,0 - l_discount@4) as volume] + │ ProjectionExec: expr=[n_name@0 as supp_nation, n_name@1 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@3 * (1 - l_discount@4) as volume] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@6, n_name@1, l_shipdate@4, l_extendedprice@2, l_discount@3] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -404,10 +404,10 @@ mod tests { │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -430,26 +430,26 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -466,13 +466,13 @@ mod tests { ┌───── Stage 9 ── tasks=3, partitions=3 │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] - │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([o_year@0], 9), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE Some(0),38,4 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@0) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume, n_name@3 as nation] + │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = BRAZIL THEN all_nations.volume ELSE 0.0000 END) as sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@0) as o_year, l_extendedprice@1 * (1 - l_discount@2) as volume, n_name@3 as nation] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[o_orderdate@3, l_extendedprice@1, l_discount@2, n_name@5] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -495,10 +495,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -520,35 +520,35 @@ mod tests { ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet:.., /testdata/tpch/plan_sf0.02/nation/10.parquet:..], [/testdata/tpch/plan_sf0.02/nation/14.parquet:.., /testdata/tpch/plan_sf0.02/nation/15.parquet:..], [/testdata/tpch/plan_sf0.02/nation/4.parquet:.., /testdata/tpch/plan_sf0.02/nation/5.parquet:.., /testdata/tpch/plan_sf0.02/nation/6.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/10.parquet:.., /testdata/tpch/plan_sf0.02/nation/11.parquet:..], [/testdata/tpch/plan_sf0.02/nation/15.parquet:.., /testdata/tpch/plan_sf0.02/nation/16.parquet:.., /testdata/tpch/plan_sf0.02/nation/2.parquet:..], [/testdata/tpch/plan_sf0.02/nation/6.parquet:.., /testdata/tpch/plan_sf0.02/nation/7.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -566,21 +566,21 @@ mod tests { #[tokio::test] async fn test_tpch_9() -> Result<(), Box> { let plan = test_tpch_query("q9").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] │ [Stage 7] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=3, partitions=3 - │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 9), input_partitions=3 │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@1) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@5 as amount] + │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@1) as o_year, l_extendedprice@2 * (1 - l_discount@3) - ps_supplycost@4 * l_quantity@5 as amount] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, o_orderdate@7, l_extendedprice@3, l_discount@4, ps_supplycost@6, l_quantity@2] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -597,10 +597,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -616,7 +616,7 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_orderdate], file_type=parquet │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_orderdate], file_type=parquet │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_orderdate], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderdate], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderdate], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -624,15 +624,15 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -650,20 +650,20 @@ mod tests { #[tokio::test] async fn test_tpch_10() -> Result<(), Box> { let plan = test_tpch_query("q10").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [revenue@2 DESC] │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=3, partitions=3 - │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] + │ SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 DESC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 9), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10, n_name@1] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 @@ -672,10 +672,10 @@ mod tests { │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -691,10 +691,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -703,7 +703,7 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -743,10 +743,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -760,10 +760,10 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 7 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([ps_partkey@0], 9), input_partitions=3 @@ -775,10 +775,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -792,10 +792,10 @@ mod tests { ┌───── Stage 6 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -810,8 +810,8 @@ mod tests { │ [Stage 3] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=3, partitions=3 - │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = 1-URGENT OR orders.o_orderpriority = 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != 1-URGENT AND orders.o_orderpriority != 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -822,10 +822,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -834,7 +834,7 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] └────────────────────────────────────────────────── "#); Ok(()) @@ -849,8 +849,8 @@ mod tests { │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=2, partitions=3 - │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + │ SortExec: expr=[count(Int64(1))@1 DESC, c_count@0 DESC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── @@ -880,10 +880,10 @@ mod tests { │ RepartitionExec: partitioning=Hash([o_custkey@1], 12), input_partitions=3 │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── "); Ok(()) @@ -895,21 +895,21 @@ mod tests { assert_snapshot!(plan, @r#" ┌───── DistributedExec │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE 0.0000 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE Some(0),38,4 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, p_type@2 as p_type] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE PROMO% THEN __common_expr_1 ELSE 0.0000 END) as sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[l_extendedprice@0 * (1 - l_discount@1) as __common_expr_1, p_type@2 as p_type] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_extendedprice@1, l_discount@2, p_type@4] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet:.., /testdata/tpch/plan_sf0.02/part/10.parquet:..], [/testdata/tpch/plan_sf0.02/part/14.parquet:.., /testdata/tpch/plan_sf0.02/part/15.parquet:..], [/testdata/tpch/plan_sf0.02/part/4.parquet:.., /testdata/tpch/plan_sf0.02/part/5.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/10.parquet:.., /testdata/tpch/plan_sf0.02/part/11.parquet:..], [/testdata/tpch/plan_sf0.02/part/15.parquet:.., /testdata/tpch/plan_sf0.02/part/16.parquet:.., /testdata/tpch/plan_sf0.02/part/2.parquet:..], [/testdata/tpch/plan_sf0.02/part/5.parquet:.., /testdata/tpch/plan_sf0.02/part/6.parquet:.., /testdata/tpch/plan_sf0.02/part/7.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet:.., /testdata/tpch/plan_sf0.02/part/10.parquet:..], [/testdata/tpch/plan_sf0.02/part/14.parquet:.., /testdata/tpch/plan_sf0.02/part/15.parquet:..], [/testdata/tpch/plan_sf0.02/part/4.parquet:.., /testdata/tpch/plan_sf0.02/part/5.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/10.parquet:.., /testdata/tpch/plan_sf0.02/part/11.parquet:..], [/testdata/tpch/plan_sf0.02/part/15.parquet:.., /testdata/tpch/plan_sf0.02/part/16.parquet:.., /testdata/tpch/plan_sf0.02/part/2.parquet:..], [/testdata/tpch/plan_sf0.02/part/5.parquet:.., /testdata/tpch/plan_sf0.02/part/6.parquet:.., /testdata/tpch/plan_sf0.02/part/7.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_type], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -918,7 +918,7 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] └────────────────────────────────────────────────── "#); Ok(()) @@ -927,22 +927,23 @@ mod tests { #[tokio::test] async fn test_tpch_15() -> Result<(), Box> { let plan = test_tpch_query("q15").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] │ [Stage 6] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 6 ── tasks=4, partitions=3 - │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 + │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: DynamicFilter [ empty ] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=1, partitions=4 │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 @@ -953,18 +954,18 @@ mod tests { ┌───── Stage 2 ── tasks=2, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=6 │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -976,13 +977,13 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -991,14 +992,14 @@ mod tests { #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { let plan = test_tpch_query("q16").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 5 ── tasks=2, partitions=3 - │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + │ SortExec: expr=[count(alias1)@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── @@ -1018,10 +1019,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1034,12 +1035,12 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) + │ FilterExec: p_brand@1 != Brand#45 AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@2 NOT LIKE MEDIUM POLISHED% │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet:.., /testdata/tpch/plan_sf0.02/part/10.parquet:..], [/testdata/tpch/plan_sf0.02/part/14.parquet:.., /testdata/tpch/plan_sf0.02/part/15.parquet:..], [/testdata/tpch/plan_sf0.02/part/4.parquet:.., /testdata/tpch/plan_sf0.02/part/5.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_type@4 NOT LIKE MEDIUM POLISHED% AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/10.parquet:.., /testdata/tpch/plan_sf0.02/part/11.parquet:..], [/testdata/tpch/plan_sf0.02/part/15.parquet:.., /testdata/tpch/plan_sf0.02/part/16.parquet:.., /testdata/tpch/plan_sf0.02/part/2.parquet:..], [/testdata/tpch/plan_sf0.02/part/5.parquet:.., /testdata/tpch/plan_sf0.02/part/6.parquet:.., /testdata/tpch/plan_sf0.02/part/7.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_type@4 NOT LIKE MEDIUM POLISHED% AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_type@4 NOT LIKE MEDIUM POLISHED% AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_type@4 NOT LIKE MEDIUM POLISHED% AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet:.., /testdata/tpch/plan_sf0.02/part/10.parquet:..], [/testdata/tpch/plan_sf0.02/part/14.parquet:.., /testdata/tpch/plan_sf0.02/part/15.parquet:..], [/testdata/tpch/plan_sf0.02/part/4.parquet:.., /testdata/tpch/plan_sf0.02/part/5.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@4 NOT LIKE MEDIUM POLISHED%, pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND (p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 49 AND 49 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 14 AND 14 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 23 AND 23 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 45 AND 45 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 19 AND 19 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 3 AND 3 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 36 AND 36 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 9 AND 9 <= p_size_max@5) AND p_type_null_count@9 != row_count@3 AND (p_type_min@7 NOT LIKE MEDIUM POLISHED% OR p_type_max@8 NOT LIKE MEDIUM POLISHED%), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/10.parquet:.., /testdata/tpch/plan_sf0.02/part/11.parquet:..], [/testdata/tpch/plan_sf0.02/part/15.parquet:.., /testdata/tpch/plan_sf0.02/part/16.parquet:.., /testdata/tpch/plan_sf0.02/part/2.parquet:..], [/testdata/tpch/plan_sf0.02/part/5.parquet:.., /testdata/tpch/plan_sf0.02/part/6.parquet:.., /testdata/tpch/plan_sf0.02/part/7.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@4 NOT LIKE MEDIUM POLISHED%, pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND (p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 49 AND 49 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 14 AND 14 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 23 AND 23 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 45 AND 45 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 19 AND 19 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 3 AND 3 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 36 AND 36 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 9 AND 9 <= p_size_max@5) AND p_type_null_count@9 != row_count@3 AND (p_type_min@7 NOT LIKE MEDIUM POLISHED% OR p_type_max@8 NOT LIKE MEDIUM POLISHED%), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@4 NOT LIKE MEDIUM POLISHED%, pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND (p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 49 AND 49 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 14 AND 14 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 23 AND 23 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 45 AND 45 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 19 AND 19 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 3 AND 3 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 36 AND 36 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 9 AND 9 <= p_size_max@5) AND p_type_null_count@9 != row_count@3 AND (p_type_min@7 NOT LIKE MEDIUM POLISHED% OR p_type_max@8 NOT LIKE MEDIUM POLISHED%), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) AND p_type@4 NOT LIKE MEDIUM POLISHED%, pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND (p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 49 AND 49 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 14 AND 14 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 23 AND 23 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 45 AND 45 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 19 AND 19 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 3 AND 3 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 36 AND 36 <= p_size_max@5 OR p_size_null_count@6 != row_count@3 AND p_size_min@4 <= 9 AND 9 <= p_size_max@5) AND p_type_null_count@9 != row_count@3 AND (p_type_min@7 NOT LIKE MEDIUM POLISHED% OR p_type_max@8 NOT LIKE MEDIUM POLISHED%), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] └────────────────────────────────────────────────── "); Ok(()) @@ -1048,32 +1049,29 @@ mod tests { #[tokio::test] async fn test_tpch_17() -> Result<(), Box> { let plan = test_tpch_query("q17").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=3 + ┌───── Stage 5 ── tasks=4, partitions=3 │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 2 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([p_partkey@2], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[l_quantity@2, l_extendedprice@3, p_partkey@0] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1084,15 +1082,21 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@3 = Brand#23 AND p_container@6 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@3 = Brand#23 AND p_container@6 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([l_partkey@0], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + ┌───── Stage 4 ── tasks=3, partitions=12 + │ RepartitionExec: partitioning=Hash([l_partkey@1], 12), input_partitions=3 + │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── + ┌───── Stage 3 ── tasks=4, partitions=9 + │ RepartitionExec: partitioning=Hash([l_partkey@0], 9), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + └────────────────────────────────────────────────── "); Ok(()) } @@ -1120,14 +1124,14 @@ mod tests { │ CoalescePartitionsExec │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=3, partitions=36 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 - │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] + │ FilterExec: sum(lineitem.l_quantity)@1 > 300.00, projection=[l_orderkey@0] │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── @@ -1138,7 +1142,7 @@ mod tests { │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_quantity], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1146,10 +1150,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1166,24 +1170,24 @@ mod tests { #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { let plan = test_tpch_query("q19").await?; - assert_snapshot!(plan, @r" + assert_snapshot!(plan, @" ┌───── DistributedExec │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalescePartitionsExec │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Some(1),20,0 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= 1.00 AND l_quantity@0 <= 11.00 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= 10.00 AND l_quantity@0 <= 20.00 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= 20.00 AND l_quantity@0 <= 30.00 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] + │ FilterExec: (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= 1.00 AND l_quantity@1 <= 11.00 OR l_quantity@1 >= 10.00 AND l_quantity@1 <= 20.00 OR l_quantity@1 >= 20.00 AND l_quantity@1 <= 30.00), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= Some(100),15,2 AND l_quantity@4 <= Some(1100),15,2 OR l_quantity@4 >= Some(1000),15,2 AND l_quantity@4 <= Some(2000),15,2 OR l_quantity@4 >= Some(2000),15,2 AND l_quantity@4 <= Some(3000),15,2) AND DynamicFilter [ empty ], pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(100),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(1100),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(1000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(2000),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(2000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(3000),15,2), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= Some(100),15,2 AND l_quantity@4 <= Some(1100),15,2 OR l_quantity@4 >= Some(1000),15,2 AND l_quantity@4 <= Some(2000),15,2 OR l_quantity@4 >= Some(2000),15,2 AND l_quantity@4 <= Some(3000),15,2) AND DynamicFilter [ empty ], pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(100),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(1100),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(1000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(2000),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(2000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(3000),15,2), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= Some(100),15,2 AND l_quantity@4 <= Some(1100),15,2 OR l_quantity@4 >= Some(1000),15,2 AND l_quantity@4 <= Some(2000),15,2 OR l_quantity@4 >= Some(2000),15,2 AND l_quantity@4 <= Some(3000),15,2) AND DynamicFilter [ empty ], pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(100),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(1100),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(1000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(2000),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(2000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(3000),15,2), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= Some(100),15,2 AND l_quantity@4 <= Some(1100),15,2 OR l_quantity@4 >= Some(1000),15,2 AND l_quantity@4 <= Some(2000),15,2 OR l_quantity@4 >= Some(2000),15,2 AND l_quantity@4 <= Some(3000),15,2) AND DynamicFilter [ empty ], pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(100),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(1100),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(1000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(2000),15,2 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= Some(2000),15,2 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= Some(3000),15,2), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= 1.00 AND l_quantity@4 <= 11.00 OR l_quantity@4 >= 10.00 AND l_quantity@4 <= 20.00 OR l_quantity@4 >= 20.00 AND l_quantity@4 <= 30.00) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 1.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 11.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 10.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 20.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 20.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 30.00), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= 1.00 AND l_quantity@4 <= 11.00 OR l_quantity@4 >= 10.00 AND l_quantity@4 <= 20.00 OR l_quantity@4 >= 20.00 AND l_quantity@4 <= 30.00) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 1.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 11.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 10.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 20.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 20.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 30.00), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= 1.00 AND l_quantity@4 <= 11.00 OR l_quantity@4 >= 10.00 AND l_quantity@4 <= 20.00 OR l_quantity@4 >= 20.00 AND l_quantity@4 <= 30.00) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 1.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 11.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 10.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 20.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 20.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 30.00), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND (l_quantity@4 >= 1.00 AND l_quantity@4 <= 11.00 OR l_quantity@4 >= 10.00 AND l_quantity@4 <= 20.00 OR l_quantity@4 >= 20.00 AND l_quantity@4 <= 30.00) AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR AND AIR <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= AIR REG AND AIR REG <= l_shipmode_max@1) AND l_shipinstruct_null_count@6 != row_count@3 AND l_shipinstruct_min@4 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@5 AND (l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 1.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 11.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 10.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 20.00 OR l_quantity_null_count@8 != row_count@3 AND l_quantity_max@7 >= 20.00 AND l_quantity_null_count@8 != row_count@3 AND l_quantity_min@9 <= 30.00), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1204,13 +1208,13 @@ mod tests { assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=3 + ┌───── Stage 8 ── tasks=4, partitions=3 │ SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true] │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=4 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([s_suppkey@0], 12), input_partitions=3 @@ -1218,10 +1222,10 @@ mod tests { │ CoalescePartitionsExec │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 1 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1232,25 +1236,22 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 + ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([ps_suppkey@0], 12), input_partitions=3 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 - │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=48 - │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + ┌───── Stage 4 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([ps_partkey@0, ps_suppkey@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)] │ CoalescePartitionsExec │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/10.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/15.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/5.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/10.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/11.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/16.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/5.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/6.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/7.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/11.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/12.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/16.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/2.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/3.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/7.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/8.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/partsupp/12.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/13.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/14.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/4.parquet:..], [/testdata/tpch/plan_sf0.02/partsupp/8.parquet:.., /testdata/tpch/plan_sf0.02/partsupp/9.parquet:..]]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 @@ -1261,16 +1262,22 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/11.parquet:.., /testdata/tpch/plan_sf0.02/part/12.parquet:..], [/testdata/tpch/plan_sf0.02/part/2.parquet:.., /testdata/tpch/plan_sf0.02/part/3.parquet:..], [/testdata/tpch/plan_sf0.02/part/7.parquet:.., /testdata/tpch/plan_sf0.02/part/8.parquet:..]]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/part/12.parquet:.., /testdata/tpch/plan_sf0.02/part/13.parquet:.., /testdata/tpch/plan_sf0.02/part/14.parquet:..], [/testdata/tpch/plan_sf0.02/part/3.parquet:.., /testdata/tpch/plan_sf0.02/part/4.parquet:..], [/testdata/tpch/plan_sf0.02/part/8.parquet:.., /testdata/tpch/plan_sf0.02/part/9.parquet:..]]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 12), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] - │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + ┌───── Stage 6 ── tasks=2, partitions=12 + │ RepartitionExec: partitioning=Hash([l_partkey@1, l_suppkey@2], 12), input_partitions=3 + │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── + ┌───── Stage 5 ── tasks=4, partitions=6 + │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + └────────────────────────────────────────────────── "); Ok(()) } @@ -1284,43 +1291,52 @@ mod tests { │ [Stage 9] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 9 ── tasks=3, partitions=3 - │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + │ SortExec: expr=[count(Int64(1))@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 8 ── tasks=4, partitions=9 │ RepartitionExec: partitioning=Hash([s_name@0], 9), input_partitions=3 │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - │ HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ HashJoinExec: mode=Partitioned, join_type=RightAnti, on=[(l_orderkey@0, l_orderkey@1)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── tasks=4, partitions=12 + ┌───── Stage 1 ── tasks=4, partitions=12 + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 12), input_partitions=3 + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([l_orderkey@1], 12), input_partitions=3 │ HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 4 ── tasks=4, partitions=12 + ┌───── Stage 5 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([l_orderkey@1], 12), input_partitions=3 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4] │ CoalescePartitionsExec - │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] │ CoalescePartitionsExec - │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] │ CoalescePartitionsExec - │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 1 ── tasks=4, partitions=48 + ┌───── Stage 2 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] │ DistributedLeafExec: @@ -1329,40 +1345,31 @@ mod tests { │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/11.parquet:.., /testdata/tpch/plan_sf0.02/nation/12.parquet:.., /testdata/tpch/plan_sf0.02/nation/13.parquet:..], [/testdata/tpch/plan_sf0.02/nation/2.parquet:.., /testdata/tpch/plan_sf0.02/nation/3.parquet:..], [/testdata/tpch/plan_sf0.02/nation/7.parquet:.., /testdata/tpch/plan_sf0.02/nation/8.parquet:..]]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/nation/13.parquet:.., /testdata/tpch/plan_sf0.02/nation/14.parquet:..], [/testdata/tpch/plan_sf0.02/nation/3.parquet:.., /testdata/tpch/plan_sf0.02/nation/4.parquet:..], [/testdata/tpch/plan_sf0.02/nation/8.parquet:.., /testdata/tpch/plan_sf0.02/nation/9.parquet:..]]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── tasks=4, partitions=48 + ┌───── Stage 3 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] │ DistributedLeafExec: │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── tasks=4, partitions=48 + ┌───── Stage 4 ── tasks=4, partitions=48 │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet:.., /testdata/tpch/plan_sf0.02/supplier/10.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/14.parquet:.., /testdata/tpch/plan_sf0.02/supplier/15.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/4.parquet:.., /testdata/tpch/plan_sf0.02/supplier/5.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/10.parquet:.., /testdata/tpch/plan_sf0.02/supplier/11.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/15.parquet:.., /testdata/tpch/plan_sf0.02/supplier/16.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/5.parquet:.., /testdata/tpch/plan_sf0.02/supplier/6.parquet:.., /testdata/tpch/plan_sf0.02/supplier/7.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/11.parquet:.., /testdata/tpch/plan_sf0.02/supplier/12.parquet:.., /testdata/tpch/plan_sf0.02/supplier/13.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/16.parquet:.., /testdata/tpch/plan_sf0.02/supplier/2.parquet:.., /testdata/tpch/plan_sf0.02/supplier/3.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/7.parquet:.., /testdata/tpch/plan_sf0.02/supplier/8.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/supplier/13.parquet:.., /testdata/tpch/plan_sf0.02/supplier/14.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/3.parquet:.., /testdata/tpch/plan_sf0.02/supplier/4.parquet:..], [/testdata/tpch/plan_sf0.02/supplier/8.parquet:.., /testdata/tpch/plan_sf0.02/supplier/9.parquet:..]]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 5 ── tasks=4, partitions=12 + ┌───── Stage 6 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([l_orderkey@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── - ┌───── Stage 7 ── tasks=4, partitions=12 - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 12), input_partitions=3 - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/10.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/14.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/15.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/4.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/5.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/10.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/11.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/16.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/5.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/6.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/7.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/11.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/12.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/13.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/16.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/2.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/3.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/7.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/8.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/lineitem/13.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/14.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/4.parquet:..], [/testdata/tpch/plan_sf0.02/lineitem/8.parquet:.., /testdata/tpch/plan_sf0.02/lineitem/9.parquet:..]]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── "); Ok(()) } @@ -1373,8 +1380,8 @@ mod tests { assert_snapshot!(plan, @" ┌───── DistributedExec │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] - │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] │ RepartitionExec: partitioning=Hash([cntrycode@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] @@ -1407,19 +1414,19 @@ mod tests { ┌───── Stage 2 ── tasks=4, partitions=12 │ RepartitionExec: partitioning=Hash([o_custkey@0], 12), input_partitions=3 │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet:.., /testdata/tpch/plan_sf0.02/orders/10.parquet:..], [/testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/15.parquet:..], [/testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/5.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/10.parquet:.., /testdata/tpch/plan_sf0.02/orders/11.parquet:..], [/testdata/tpch/plan_sf0.02/orders/15.parquet:.., /testdata/tpch/plan_sf0.02/orders/16.parquet:.., /testdata/tpch/plan_sf0.02/orders/2.parquet:..], [/testdata/tpch/plan_sf0.02/orders/5.parquet:.., /testdata/tpch/plan_sf0.02/orders/6.parquet:.., /testdata/tpch/plan_sf0.02/orders/7.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/11.parquet:.., /testdata/tpch/plan_sf0.02/orders/12.parquet:.., /testdata/tpch/plan_sf0.02/orders/13.parquet:..], [/testdata/tpch/plan_sf0.02/orders/2.parquet:.., /testdata/tpch/plan_sf0.02/orders/3.parquet:..], [/testdata/tpch/plan_sf0.02/orders/7.parquet:.., /testdata/tpch/plan_sf0.02/orders/8.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/orders/13.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:.., /testdata/tpch/plan_sf0.02/orders/14.parquet:..], [/testdata/tpch/plan_sf0.02/orders/3.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:.., /testdata/tpch/plan_sf0.02/orders/4.parquet:..], [/testdata/tpch/plan_sf0.02/orders/8.parquet:.., /testdata/tpch/plan_sf0.02/orders/9.parquet:..]]}, projection=[o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible └────────────────────────────────────────────────── ┌───── Stage 4 ── tasks=4, partitions=12 │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] + │ FilterExec: c_acctbal@1 > 0.00 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] │ DistributedLeafExec: - │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > Some(0),15,2 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] - │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > Some(0),15,2 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] - │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > Some(0),15,2 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] - │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > Some(0),15,2 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet:.., /testdata/tpch/plan_sf0.02/customer/10.parquet:..], [/testdata/tpch/plan_sf0.02/customer/14.parquet:.., /testdata/tpch/plan_sf0.02/customer/15.parquet:..], [/testdata/tpch/plan_sf0.02/customer/4.parquet:.., /testdata/tpch/plan_sf0.02/customer/5.parquet:.., /testdata/tpch/plan_sf0.02/customer/6.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > 0.00 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > 0.00, required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/10.parquet:.., /testdata/tpch/plan_sf0.02/customer/11.parquet:..], [/testdata/tpch/plan_sf0.02/customer/15.parquet:.., /testdata/tpch/plan_sf0.02/customer/16.parquet:..], [/testdata/tpch/plan_sf0.02/customer/6.parquet:.., /testdata/tpch/plan_sf0.02/customer/7.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > 0.00 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > 0.00, required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/11.parquet:.., /testdata/tpch/plan_sf0.02/customer/12.parquet:.., /testdata/tpch/plan_sf0.02/customer/13.parquet:..], [/testdata/tpch/plan_sf0.02/customer/16.parquet:.., /testdata/tpch/plan_sf0.02/customer/2.parquet:.., /testdata/tpch/plan_sf0.02/customer/3.parquet:..], [/testdata/tpch/plan_sf0.02/customer/7.parquet:.., /testdata/tpch/plan_sf0.02/customer/8.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > 0.00 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > 0.00, required_guarantees=[] + │ t3: DataSourceExec: file_groups={3 groups: [[/testdata/tpch/plan_sf0.02/customer/13.parquet:.., /testdata/tpch/plan_sf0.02/customer/14.parquet:..], [/testdata/tpch/plan_sf0.02/customer/3.parquet:.., /testdata/tpch/plan_sf0.02/customer/4.parquet:..], [/testdata/tpch/plan_sf0.02/customer/8.parquet:.., /testdata/tpch/plan_sf0.02/customer/9.parquet:..]]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > 0.00 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > 0.00, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) diff --git a/tests/udfs.rs b/tests/udfs.rs index 109aa8bd1..4c17e53ac 100644 --- a/tests/udfs.rs +++ b/tests/udfs.rs @@ -42,8 +42,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[count(*)@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[test_udf(weather.RainToday)@0 as test_udf(weather.RainToday), count(Int64(1))@1 as count(*)] + │ ProjectionExec: expr=[test_udf(weather.RainToday)@0 as test_udf(weather.RainToday), count(Int64(1))@1 as count(*)] + │ SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[test_udf(weather.RainToday)@0 as test_udf(weather.RainToday)], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── diff --git a/tests/work_unit_feed.rs b/tests/work_unit_feed.rs index 144684134..6721988df 100644 --- a/tests/work_unit_feed.rs +++ b/tests/work_unit_feed.rs @@ -225,8 +225,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[letter@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as cnt, letter@0 as letter] + │ ProjectionExec: expr=[count(Int64(1))@1 as cnt, letter@0 as letter] + │ SortExec: expr=[letter@0 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[letter@0 as letter], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── @@ -272,8 +272,8 @@ mod tests { │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 3 ── tasks=2, partitions=3 - │ SortExec: expr=[a_task@0 ASC NULLS LAST, b_letter@3 ASC NULLS LAST, b_task@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[task@2 as a_task, letter@3 as a_letter, task@0 as b_task, letter@1 as b_letter] + │ ProjectionExec: expr=[task@2 as a_task, letter@3 as a_letter, task@0 as b_task, letter@1 as b_letter] + │ SortExec: expr=[task@2 ASC NULLS LAST, letter@1 ASC NULLS LAST, task@0 ASC NULLS LAST], preserve_partitioning=[true] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(letter@1, letter@1)] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -521,14 +521,14 @@ mod tests { ) .await?; - assert_snapshot!(plan + &results, @r" + assert_snapshot!(plan + &results, @" ┌───── DistributedExec │ SortPreservingMergeExec: [tag@0 ASC NULLS LAST, letter@1 ASC NULLS LAST] │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 2 ── tasks=2, partitions=3 - │ SortExec: expr=[tag@0 ASC NULLS LAST, letter@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[tag@0 as tag, letter@1 as letter, count(Int64(1))@2 as cnt] + │ ProjectionExec: expr=[tag@0 as tag, letter@1 as letter, count(Int64(1))@2 as cnt] + │ SortExec: expr=[tag@0 ASC NULLS LAST, letter@1 ASC NULLS LAST], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[tag@0 as tag, letter@1 as letter], aggr=[count(Int64(1))] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── From 7d78d96d0b96b6370dec55756c21ec6c80bea5a5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 29 Jun 2026 18:11:48 -0700 Subject: [PATCH 03/15] Moved the worker-protocol messages out from behind the grpc feature. The prost message types carry no tonic dependency, so a transport that is not gRPC can speak the same wire shape without pulling in the gRPC stack. Only the tonic client and server stay gated; the generator emits those gates so a regeneration cannot drop them. tonic-prost feeds only the generated client and server, so it moves behind the feature too. Co-authored-by: Stu Hood --- Cargo.toml | 3 ++- src/lib.rs | 5 +++++ src/protocol/generated/mod.rs | 1 + src/protocol/{grpc => }/generated/worker.rs | 2 ++ src/protocol/grpc/gen/src/main.rs | 6 +++++- src/protocol/grpc/generated/mod.rs | 1 - src/protocol/grpc/metrics_proto.rs | 2 +- src/protocol/grpc/mod.rs | 1 - src/protocol/grpc/observability/gen/src/main.rs | 2 +- src/protocol/grpc/observability/generated/observability.rs | 2 +- src/protocol/grpc/observability/service.rs | 2 +- src/protocol/grpc/worker_client.rs | 4 ++-- src/protocol/grpc/worker_service.rs | 2 +- src/protocol/mod.rs | 4 ++++ 14 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 src/protocol/generated/mod.rs rename src/protocol/{grpc => }/generated/worker.rs (99%) delete mode 100644 src/protocol/grpc/generated/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 27843802e..3bc00e8ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ sysinfo = { version = "0.30", optional = true } sketches-ddsketch = { version = "0.3", features = ["use_serde"] } num-traits = "0.2" bincode = "1" -tonic-prost = "0.14.2" +tonic-prost = { version = "0.14.2", optional = true } # grpc-specific features arrow-flight = { version = "59", optional = true } @@ -76,6 +76,7 @@ grpc = [ "arrow-select", "arrow-ipc", "tonic", + "tonic-prost", "tower", ] diff --git a/src/lib.rs b/src/lib.rs index 561799e46..e42652e97 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,11 @@ pub mod test_utils; #[cfg(feature = "grpc")] pub use protocol::grpc; +/// The worker-protocol prost message types, independent of any transport. A non-gRPC transport +/// reaches for these to speak the same wire shape the gRPC path serializes. Unstable: this +/// mirrors `worker.proto`, which is regenerated freely. +pub use protocol::generated::worker as proto; + pub use codec::DistributedCodec; pub use worker_resolver::{WorkerResolver, get_distributed_worker_resolver}; diff --git a/src/protocol/generated/mod.rs b/src/protocol/generated/mod.rs new file mode 100644 index 000000000..2c8b83993 --- /dev/null +++ b/src/protocol/generated/mod.rs @@ -0,0 +1 @@ +pub mod worker; diff --git a/src/protocol/grpc/generated/worker.rs b/src/protocol/generated/worker.rs similarity index 99% rename from src/protocol/grpc/generated/worker.rs rename to src/protocol/generated/worker.rs index 4441b67fa..aaaf87ea0 100644 --- a/src/protocol/grpc/generated/worker.rs +++ b/src/protocol/generated/worker.rs @@ -474,6 +474,7 @@ pub struct MaxGauge { pub value: u64, } /// Generated client implementations. +#[cfg(feature = "grpc")] pub mod worker_service_client { #![allow( unused_variables, @@ -623,6 +624,7 @@ pub mod worker_service_client { } } /// Generated server implementations. +#[cfg(feature = "grpc")] pub mod worker_service_server { #![allow( unused_variables, diff --git a/src/protocol/grpc/gen/src/main.rs b/src/protocol/grpc/gen/src/main.rs index 7505aae5f..cde0e4e6c 100644 --- a/src/protocol/grpc/gen/src/main.rs +++ b/src/protocol/grpc/gen/src/main.rs @@ -6,7 +6,7 @@ fn main() -> Result<(), Box> { let proto_dir = repo_root.join("src/protocol/grpc"); let proto_file = proto_dir.join("worker.proto"); - let out_dir = repo_root.join("src/protocol/grpc/generated"); + let out_dir = repo_root.join("src/protocol/generated"); fs::create_dir_all(&out_dir)?; @@ -18,6 +18,10 @@ fn main() -> Result<(), Box> { tonic_prost_build::configure() .build_server(true) .build_client(true) + // The generated messages build with `grpc` off; only the tonic client and server carry + // the feature gate. Emitted here so a regeneration cannot drop the gates. + .client_mod_attribute(".", "#[cfg(feature = \"grpc\")]") + .server_mod_attribute(".", "#[cfg(feature = \"grpc\")]") .out_dir(&out_dir) .extern_path(".worker.FlightData", "::arrow_flight::FlightData") .extern_path( diff --git a/src/protocol/grpc/generated/mod.rs b/src/protocol/grpc/generated/mod.rs deleted file mode 100644 index 844c269cb..000000000 --- a/src/protocol/grpc/generated/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod worker; diff --git a/src/protocol/grpc/metrics_proto.rs b/src/protocol/grpc/metrics_proto.rs index f4eeda017..9dfc145ed 100644 --- a/src/protocol/grpc/metrics_proto.rs +++ b/src/protocol/grpc/metrics_proto.rs @@ -1,4 +1,4 @@ -use super::generated::worker as pb; +use crate::protocol::generated::worker as pb; use chrono::DateTime; use datafusion::common::internal_err; use datafusion::error::DataFusionError; diff --git a/src/protocol/grpc/mod.rs b/src/protocol/grpc/mod.rs index e432b607b..fc367585f 100644 --- a/src/protocol/grpc/mod.rs +++ b/src/protocol/grpc/mod.rs @@ -1,6 +1,5 @@ mod channel_resolver; mod errors; -mod generated; mod metrics_proto; mod observability; mod on_drop_stream; diff --git a/src/protocol/grpc/observability/gen/src/main.rs b/src/protocol/grpc/observability/gen/src/main.rs index 124f0f1bc..1b55fb7dd 100644 --- a/src/protocol/grpc/observability/gen/src/main.rs +++ b/src/protocol/grpc/observability/gen/src/main.rs @@ -21,7 +21,7 @@ fn main() -> Result<(), Box> { .out_dir(&out_dir) .extern_path( ".observability.TaskKey", - "crate::protocol::grpc::generated::worker::TaskKey", + "crate::protocol::generated::worker::TaskKey", ) .compile_protos(&[proto_file], &[proto_dir])?; diff --git a/src/protocol/grpc/observability/generated/observability.rs b/src/protocol/grpc/observability/generated/observability.rs index 3b0a95cfe..98aa592df 100644 --- a/src/protocol/grpc/observability/generated/observability.rs +++ b/src/protocol/grpc/observability/generated/observability.rs @@ -12,7 +12,7 @@ pub struct GetTaskProgressRequest {} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct TaskProgress { #[prost(message, optional, tag = "1")] - pub task_key: ::core::option::Option, + pub task_key: ::core::option::Option, #[prost(enumeration = "TaskStatus", tag = "4")] pub status: i32, #[prost(uint64, tag = "5")] diff --git a/src/protocol/grpc/observability/service.rs b/src/protocol/grpc/observability/service.rs index 27edee2e3..04c2aa26d 100644 --- a/src/protocol/grpc/observability/service.rs +++ b/src/protocol/grpc/observability/service.rs @@ -4,7 +4,7 @@ use super::{ }; use crate::common::serialize_uuid; use crate::grpc::{GetClusterWorkersRequest, GetClusterWorkersResponse}; -use crate::protocol::grpc::generated::worker as worker_pb; +use crate::protocol::generated::worker as worker_pb; use crate::worker::{SingleWriteMultiRead, TaskData}; use crate::{TaskKey, WorkerResolver}; use datafusion::error::DataFusionError; diff --git a/src/protocol/grpc/worker_client.rs b/src/protocol/grpc/worker_client.rs index 566126715..d342eebe1 100644 --- a/src/protocol/grpc/worker_client.rs +++ b/src/protocol/grpc/worker_client.rs @@ -1,10 +1,10 @@ use super::channel_resolver::BoxCloneSyncChannel; use super::errors::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; -use super::generated::worker as pb; use super::metrics_proto::metrics_set_proto_to_df; use crate::common::serialize_uuid; -use crate::grpc::generated::worker::FlightAppMetadata; use crate::grpc::on_drop_stream::on_drop_stream; +use crate::protocol::generated::worker as pb; +use crate::protocol::generated::worker::FlightAppMetadata; use crate::{ BytesMetricExt, CoordinatorToWorkerMsg, DistributedConfig, ExecuteTaskRequest, FirstLatencyMetric, GetWorkerInfoRequest, GetWorkerInfoResponse, LatencyMetricExt, LoadInfo, diff --git a/src/protocol/grpc/worker_service.rs b/src/protocol/grpc/worker_service.rs index 6fba14ee5..831865a9b 100644 --- a/src/protocol/grpc/worker_service.rs +++ b/src/protocol/grpc/worker_service.rs @@ -1,7 +1,7 @@ use super::errors::{datafusion_error_to_tonic_status, map_status_to_datafusion_error}; -use super::generated::worker as pb; use super::metrics_proto::df_metrics_set_to_proto; use super::spawn_select_all::spawn_select_all; +use crate::protocol::generated::worker as pb; use crate::common::{deserialize_uuid, now_ns}; use crate::protocol::ProducerHeadSpec; diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 6df0e11b1..024658766 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -2,6 +2,10 @@ pub mod grpc; mod channel_resolver; +// The prost message types carry no tonic dependency, so a non-gRPC transport (an in-process +// worker, a shared-memory mesh) can speak the same wire shape without pulling in the whole gRPC +// stack. +pub(crate) mod generated; mod worker_channel; pub(crate) use channel_resolver::set_distributed_channel_resolver; From 31eec847a69f586e4d0473b126980c56158c8af4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 29 Jun 2026 18:46:51 -0700 Subject: [PATCH 04/15] Made the no-gRPC build compile and its unit suite run in CI. The benchmarks crate's dev-dependency on the lib re-unified grpc into every test build, so a genuine no-gRPC test run was impossible; the dataset suites move into the benchmarks crate and the gRPC-coupled test utilities gate behind grpc. A unit-test-no-grpc job then runs the whole lib suite with the feature off. Co-authored-by: Stu Hood --- .github/workflows/ci.yml | 31 ++- Cargo.lock | 3 +- Cargo.toml | 42 ++- benchmarks/Cargo.toml | 11 + .../tests}/clickbench_correctness_test.rs | 2 +- .../tests}/clickbench_plans_test.rs | 2 +- .../tests}/stateful_data_cleanup.rs | 2 +- .../tests}/tpcds_correctness_test.rs | 2 +- .../tests}/tpcds_plans_test.rs | 2 +- .../tests}/tpch_correctness_test.rs | 2 +- .../tests}/tpch_plans_test.rs | 2 +- src/coordinator/metrics_store.rs | 2 +- src/execution_plans/metrics.rs | 2 +- src/execution_plans/mod.rs | 4 +- src/lib.rs | 2 +- src/metrics/task_metrics_collector.rs | 3 +- src/metrics/task_metrics_rewriter.rs | 3 +- src/test_utils/in_memory_channel_resolver.rs | 254 ++++++++++-------- src/test_utils/mod.rs | 1 + src/test_utils/routing.rs | 2 +- src/worker/mod.rs | 4 +- 21 files changed, 228 insertions(+), 150 deletions(-) rename {tests => benchmarks/tests}/clickbench_correctness_test.rs (99%) rename {tests => benchmarks/tests}/clickbench_plans_test.rs (99%) rename {tests => benchmarks/tests}/stateful_data_cleanup.rs (98%) rename {tests => benchmarks/tests}/tpcds_correctness_test.rs (99%) rename {tests => benchmarks/tests}/tpcds_plans_test.rs (99%) rename {tests => benchmarks/tests}/tpch_correctness_test.rs (99%) rename {tests => benchmarks/tests}/tpch_plans_test.rs (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c5da3fde..5748469b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,17 @@ jobs: - uses: ./.github/actions/setup - run: cargo test --features integration + # Runs the whole lib unit suite with `grpc` off, the proof that the abstraction builds and runs + # without gRPC. `lfs: true` because some planner unit tests load parquet testdata. + unit-test-no-grpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - uses: ./.github/actions/setup + - run: cargo test --no-default-features --lib + tpch-correctness-test: runs-on: ubuntu-latest strategy: @@ -50,7 +61,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test --features tpch --test tpch_correctness_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_correctness_test env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -59,7 +70,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test --features tpch --test tpch_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_plans_test tpcds-correctness-test: runs-on: ubuntu-latest @@ -73,9 +84,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/tpcds/main.zip + path: benchmarks/testdata/tpcds/main.zip key: "main.zip" - - run: cargo test --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }} + - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }} env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -86,9 +97,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/tpcds/main.zip + path: benchmarks/testdata/tpcds/main.zip key: "main.zip" - - run: cargo test --features tpcds --test tpcds_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_plans_test clickbench-correctness-test: runs-on: ubuntu-latest @@ -101,9 +112,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/clickbench/ + path: benchmarks/testdata/clickbench/ key: "data" - - run: cargo test --features clickbench --test clickbench_correctness_test + - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_correctness_test env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -114,9 +125,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/clickbench/ + path: benchmarks/testdata/clickbench/ key: "data" - - run: cargo test --features clickbench --test clickbench_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_plans_test format-check: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 08320584b..346b3aed8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2108,7 +2108,6 @@ dependencies = [ "crossbeam-queue", "dashmap", "datafusion", - "datafusion-distributed-benchmarks", "datafusion-proto", "delegate", "futures", @@ -2151,12 +2150,14 @@ dependencies = [ "futures", "log", "parquet", + "pretty_assertions", "reqwest", "serde", "serde_json", "sketches-ddsketch", "structopt", "sysinfo", + "test-case", "tokio", "tonic", "tpchgen", diff --git a/Cargo.toml b/Cargo.toml index 3bc00e8ec..c38bc079c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,14 +84,9 @@ integration = ["insta", "dep:parquet", "arrow", "hyper-util", "grpc", "parquet", system-metrics = ["sysinfo"] -tpch = ["integration"] -tpcds = ["integration"] -clickbench = ["integration"] -slow-tests = [] sysinfo = ["dep:sysinfo"] [dev-dependencies] -datafusion-distributed-benchmarks = { path = "benchmarks" } # The crate's own tests drive `SessionContext::sql` and register parquet tables, so they need # both features regardless of which ones a consumer selects. Requesting them here keeps # `--no-default-features` test runs compiling without forcing either on library consumers. @@ -106,6 +101,43 @@ hyper-util = "0.1.16" pretty_assertions = "1.4" test-case = "3.3.1" +# Every example drives the gRPC transport, so a no-grpc build skips them. +[[example]] +name = "custom_distributed_partial_reduction_tree" +required-features = ["integration"] + +[[example]] +name = "custom_execution_plan" +required-features = ["integration"] + +[[example]] +name = "custom_worker_url_routing" +required-features = ["integration"] + +[[example]] +name = "work_unit_feed" +required-features = ["integration"] + +[[example]] +name = "in_memory_cluster" +required-features = ["grpc"] + +[[example]] +name = "localhost_run" +required-features = ["grpc"] + +[[example]] +name = "localhost_versioned_run" +required-features = ["grpc"] + +[[example]] +name = "localhost_worker" +required-features = ["grpc"] + +[[example]] +name = "localhost_versioned_worker" +required-features = ["grpc"] + [workspace.lints.clippy] disallowed_types = "deny" disallowed-methods = "deny" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 13f795087..50361580a 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -26,9 +26,20 @@ reqwest = "0.12" zip = "6.0" sketches-ddsketch = "0.3" +[features] +# Gates for the dataset test suites under `tests/`. They live here rather than in the library +# because, as a dev-dependency of the library, this crate re-enables `grpc` on every test build +# through feature unification, which made the no-grpc config untestable. +tpch = [] +tpcds = [] +clickbench = [] +slow-tests = [] + [dev-dependencies] criterion = "0.5" sysinfo = "0.30" +pretty_assertions = "1.4" +test-case = "3.3.1" [build-dependencies] built = { version = "0.8", features = ["git2", "chrono"] } diff --git a/tests/clickbench_correctness_test.rs b/benchmarks/tests/clickbench_correctness_test.rs similarity index 99% rename from tests/clickbench_correctness_test.rs rename to benchmarks/tests/clickbench_correctness_test.rs index acf3daa85..25fb38f8b 100644 --- a/tests/clickbench_correctness_test.rs +++ b/benchmarks/tests/clickbench_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "clickbench", test))] +#[cfg(all(feature = "clickbench", test))] mod tests { use datafusion::arrow::array::RecordBatch; use datafusion::common::plan_err; diff --git a/tests/clickbench_plans_test.rs b/benchmarks/tests/clickbench_plans_test.rs similarity index 99% rename from tests/clickbench_plans_test.rs rename to benchmarks/tests/clickbench_plans_test.rs index cd1f76d3e..7b1dee5ea 100644 --- a/tests/clickbench_plans_test.rs +++ b/benchmarks/tests/clickbench_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "clickbench", test))] +#[cfg(all(feature = "clickbench", test))] mod tests { use datafusion::error::Result; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; diff --git a/tests/stateful_data_cleanup.rs b/benchmarks/tests/stateful_data_cleanup.rs similarity index 98% rename from tests/stateful_data_cleanup.rs rename to benchmarks/tests/stateful_data_cleanup.rs index fae892d27..f977c61ac 100644 --- a/tests/stateful_data_cleanup.rs +++ b/benchmarks/tests/stateful_data_cleanup.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion::common::instant::Instant; use datafusion::error::Result; diff --git a/tests/tpcds_correctness_test.rs b/benchmarks/tests/tpcds_correctness_test.rs similarity index 99% rename from tests/tpcds_correctness_test.rs rename to benchmarks/tests/tpcds_correctness_test.rs index e4baeba9d..8d5d87220 100644 --- a/tests/tpcds_correctness_test.rs +++ b/benchmarks/tests/tpcds_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpcds", test))] +#[cfg(all(feature = "tpcds", test))] mod tests { use datafusion::arrow::array::RecordBatch; use datafusion::common::plan_err; diff --git a/tests/tpcds_plans_test.rs b/benchmarks/tests/tpcds_plans_test.rs similarity index 99% rename from tests/tpcds_plans_test.rs rename to benchmarks/tests/tpcds_plans_test.rs index 12bf66524..dea57e4d3 100644 --- a/tests/tpcds_plans_test.rs +++ b/benchmarks/tests/tpcds_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpcds", test))] +#[cfg(all(feature = "tpcds", test))] mod tests { use datafusion::error::Result; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; diff --git a/tests/tpch_correctness_test.rs b/benchmarks/tests/tpch_correctness_test.rs similarity index 99% rename from tests/tpch_correctness_test.rs rename to benchmarks/tests/tpch_correctness_test.rs index 3ae089dc6..7d6ab2ce8 100644 --- a/tests/tpch_correctness_test.rs +++ b/benchmarks/tests/tpch_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; diff --git a/tests/tpch_plans_test.rs b/benchmarks/tests/tpch_plans_test.rs similarity index 99% rename from tests/tpch_plans_test.rs rename to benchmarks/tests/tpch_plans_test.rs index 70fa5e54a..cbbd4da67 100644 --- a/tests/tpch_plans_test.rs +++ b/benchmarks/tests/tpch_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ diff --git a/src/coordinator/metrics_store.rs b/src/coordinator/metrics_store.rs index 8ccc1f7c0..ed55db2c3 100644 --- a/src/coordinator/metrics_store.rs +++ b/src/coordinator/metrics_store.rs @@ -27,7 +27,7 @@ impl MetricsStore { self.rx.borrow().get(key).cloned() } - #[cfg(test)] + #[cfg(all(test, feature = "grpc"))] pub(crate) fn from_entries(entries: impl IntoIterator) -> Self { let map: HashMap<_, _> = entries.into_iter().collect(); let (tx, rx) = watch::channel(map); diff --git a/src/execution_plans/metrics.rs b/src/execution_plans/metrics.rs index 0fefdf3e2..de97ed049 100644 --- a/src/execution_plans/metrics.rs +++ b/src/execution_plans/metrics.rs @@ -21,7 +21,7 @@ impl MetricsWrapperExec { Self { inner, metrics } } - #[cfg(test)] + #[cfg(all(test, feature = "grpc"))] pub(crate) fn inner(&self) -> &Arc { &self.inner } diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index ecdcbc35d..97904ef14 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -8,7 +8,9 @@ mod network_coalesce; mod network_shuffle; mod sampler; -#[cfg(any(test, feature = "integration"))] +// The benchmark fixtures spin up real gRPC workers (`tonic` servers, Flight channels), so they +// only exist when that transport is compiled in. +#[cfg(all(feature = "grpc", any(test, feature = "integration")))] pub mod benchmarks; pub use broadcast::BroadcastExec; diff --git a/src/lib.rs b/src/lib.rs index e42652e97..d8d1f88fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,7 +73,7 @@ pub use worker::{ Worker, WorkerQueryContext, WorkerSessionBuilder, }; -#[cfg(any(feature = "integration", test))] +#[cfg(all(feature = "grpc", any(feature = "integration", test)))] pub use execution_plans::benchmarks::{ LocalRepartitionBench, LocalRepartitionFixture, LocalRepartitionMode, ShuffleBench, ShuffleFixture, TransportBench, TransportBenchMode, TransportFixture, diff --git a/src/metrics/task_metrics_collector.rs b/src/metrics/task_metrics_collector.rs index ba6e68556..f0647b95b 100644 --- a/src/metrics/task_metrics_collector.rs +++ b/src/metrics/task_metrics_collector.rs @@ -19,7 +19,8 @@ pub fn collect_plan_metrics(plan: &Arc) -> Result Self { - Self::from_configured_worker(builder, |worker| worker) - } - - /// Build an [InMemoryChannelResolver] with a custom [WorkerSessionBuilder] and worker setup. - pub fn from_configured_worker( - builder: impl WorkerSessionBuilder + Send + Sync + 'static, - configure_worker: impl Fn(Worker) -> Worker + Send + Sync + 'static, - ) -> Self { - let (client, server) = tokio::io::duplex(1024 * 1024); - - let mut client = Some(client); - let channel = Endpoint::try_from(DUMMY_URL) - .expect("Invalid dummy URL for building an endpoint. This should never happen") - .connect_with_connector_lazy(tower::service_fn(move |_| { - let client = client - .take() - .expect("Client taken twice. This should never happen"); - async move { Ok::<_, std::io::Error>(TokioIo::new(client)) } - })); - - let this = Self { - channel: grpc::BoxCloneSyncChannel::new(channel), - }; - let this_clone = this.clone(); - - let endpoint = Worker::from_session_builder(builder.map(move |builder| { - let this = this.clone(); - Ok(builder.with_distributed_channel_resolver(this).build()) - })); - let endpoint = configure_worker(endpoint); - - #[allow(clippy::disallowed_methods)] - tokio::spawn(async move { - Server::builder() - .add_service(endpoint.into_worker_server()) - .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) - .await - }); - - this_clone - } -} - -impl Default for InMemoryChannelResolver { - fn default() -> Self { - Self::from_session_builder(DefaultSessionBuilder) - } -} - -#[async_trait] -impl ChannelResolver for InMemoryChannelResolver { - async fn get_worker_client_for_url( - &self, - _: &url::Url, - ) -> Result, DataFusionError> { - Ok(grpc::create_worker_client(self.channel.clone())) - } -} - pub struct InMemoryWorkerResolver { n_workers: usize, } @@ -105,41 +23,139 @@ impl WorkerResolver for InMemoryWorkerResolver { } } -/// Creates a distributed session context backed by a single in-memory worker service. -/// The set of produced worker URLs is deterministic, taking the form http://worker-. -pub async fn start_in_memory_context( - num_workers: usize, - session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, -) -> SessionContext { - let channel_resolver = InMemoryChannelResolver::from_session_builder(session_builder); - let state = SessionStateBuilder::new() - .with_default_features() - .with_distributed_planner() - .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) - .with_distributed_channel_resolver(channel_resolver) - .build(); - SessionContext::from(state) -} +// The channel resolver runs a real `tonic` worker server over a tokio duplex, so it only compiles +// with the gRPC transport. The worker resolver above stays available without it, so the planner +// tests that only need worker URLs keep building in a no-gRPC config. +#[cfg(feature = "grpc")] +mod channel { + use super::InMemoryWorkerResolver; + use crate::{ + ChannelResolver, DefaultSessionBuilder, DistributedExt, MappedWorkerSessionBuilderExt, + SessionStateBuilderExt, Worker, WorkerChannel, WorkerSessionBuilder, grpc, + }; + use async_trait::async_trait; + use datafusion::common::DataFusionError; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::{SessionConfig, SessionContext}; + use hyper_util::rt::TokioIo; + use tonic::transport::{Endpoint, Server}; + + const DUMMY_URL: &str = "http://localhost:50051"; + + /// [ChannelResolver] implementation that returns gRPC clients backed by an in-memory + /// tokio duplex rather than a TCP connection. + #[derive(Clone)] + pub struct InMemoryChannelResolver { + channel: grpc::BoxCloneSyncChannel, + } + + impl InMemoryChannelResolver { + /// Build an [InMemoryChannelResolver] with a custom [WorkerSessionBuilder]. + /// This allows you to inject your own DataFusion extensions in the in-memory worker + /// spawned by this method. + pub fn from_session_builder( + builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> Self { + Self::from_configured_worker(builder, |worker| worker) + } + + /// Build an [InMemoryChannelResolver] with a custom [WorkerSessionBuilder] and worker setup. + pub fn from_configured_worker( + builder: impl WorkerSessionBuilder + Send + Sync + 'static, + configure_worker: impl Fn(Worker) -> Worker + Send + Sync + 'static, + ) -> Self { + let (client, server) = tokio::io::duplex(1024 * 1024); + + let mut client = Some(client); + let channel = Endpoint::try_from(DUMMY_URL) + .expect("Invalid dummy URL for building an endpoint. This should never happen") + .connect_with_connector_lazy(tower::service_fn(move |_| { + let client = client + .take() + .expect("Client taken twice. This should never happen"); + async move { Ok::<_, std::io::Error>(TokioIo::new(client)) } + })); + + let this = Self { + channel: grpc::BoxCloneSyncChannel::new(channel), + }; + let this_clone = this.clone(); + + let endpoint = Worker::from_session_builder(builder.map(move |builder| { + let this = this.clone(); + Ok(builder.with_distributed_channel_resolver(this).build()) + })); + let endpoint = configure_worker(endpoint); + + #[allow(clippy::disallowed_methods)] + tokio::spawn(async move { + Server::builder() + .add_service(endpoint.into_worker_server()) + .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) + .await + }); + + this_clone + } + } + + impl Default for InMemoryChannelResolver { + fn default() -> Self { + Self::from_session_builder(DefaultSessionBuilder) + } + } -/// Creates a distributed session context backed by a configurable in-memory worker service. -/// -/// Like [crate::test_utils::localhost::start_localhost_context], this uses tiny file-scan -/// partitions so small test datasets still cross worker boundaries. -pub async fn start_configured_in_memory_context( - num_workers: usize, - session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, - configure_worker: impl Fn(Worker) -> Worker + Send + Sync + 'static, -) -> SessionContext { - let channel_resolver = - InMemoryChannelResolver::from_configured_worker(session_builder, configure_worker); - let state = SessionStateBuilder::new() - .with_default_features() - .with_config(SessionConfig::new().with_target_partitions(num_workers)) - .with_distributed_planner() - .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) - .with_distributed_channel_resolver(channel_resolver) - .with_distributed_file_scan_config_bytes_per_partition(1) - .unwrap() - .build(); - SessionContext::from(state) + #[async_trait] + impl ChannelResolver for InMemoryChannelResolver { + async fn get_worker_client_for_url( + &self, + _: &url::Url, + ) -> Result, DataFusionError> { + Ok(grpc::create_worker_client(self.channel.clone())) + } + } + + /// Creates a distributed session context backed by a single in-memory worker service. + /// The set of produced worker URLs is deterministic, taking the form http://worker-. + pub async fn start_in_memory_context( + num_workers: usize, + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> SessionContext { + let channel_resolver = InMemoryChannelResolver::from_session_builder(session_builder); + let state = SessionStateBuilder::new() + .with_default_features() + .with_distributed_planner() + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) + .with_distributed_channel_resolver(channel_resolver) + .build(); + SessionContext::from(state) + } + + /// Creates a distributed session context backed by a configurable in-memory worker service. + /// + /// Like [crate::test_utils::localhost::start_localhost_context], this uses tiny file-scan + /// partitions so small test datasets still cross worker boundaries. + pub async fn start_configured_in_memory_context( + num_workers: usize, + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, + configure_worker: impl Fn(Worker) -> Worker + Send + Sync + 'static, + ) -> SessionContext { + let channel_resolver = + InMemoryChannelResolver::from_configured_worker(session_builder, configure_worker); + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::new().with_target_partitions(num_workers)) + .with_distributed_planner() + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) + .with_distributed_channel_resolver(channel_resolver) + .with_distributed_file_scan_config_bytes_per_partition(1) + .unwrap() + .build(); + SessionContext::from(state) + } } + +#[cfg(feature = "grpc")] +pub use channel::{ + InMemoryChannelResolver, start_configured_in_memory_context, start_in_memory_context, +}; diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index dc66f9770..6c84a3ff6 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,5 +1,6 @@ pub mod in_memory_channel_resolver; pub mod insta; +#[cfg(feature = "grpc")] pub mod localhost; pub mod metrics; pub mod mock_exec; diff --git a/src/test_utils/routing.rs b/src/test_utils/routing.rs index 7a1df81b6..9f515e57a 100644 --- a/src/test_utils/routing.rs +++ b/src/test_utils/routing.rs @@ -2,6 +2,7 @@ use arrow::{ array::{Int64Array, RecordBatch, StringArray}, datatypes::{DataType, Field, Schema, SchemaRef}, }; +use async_trait::async_trait; use datafusion::{ catalog::{Session, TableFunctionImpl, TableProvider}, common::{Result, ScalarValue, Statistics, internal_err, plan_err}, @@ -21,7 +22,6 @@ use datafusion_proto::{ use futures::stream; use prost::Message; use std::{fmt::Formatter, sync::Arc}; -use tonic::async_trait; use crate::execution_plans::DistributedLeafExec; use crate::worker::LocalWorkerContext; diff --git a/src/worker/mod.rs b/src/worker/mod.rs index b2e79eac3..030ac357f 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -3,7 +3,9 @@ mod impl_execute_task; mod session_builder; mod single_write_multi_read; mod task_data; -#[cfg(any(test, feature = "integration"))] +// `worker_handles` builds `tonic` servers and Flight channels for the benchmark fixtures, which +// only compile with the gRPC transport. +#[cfg(all(feature = "grpc", any(test, feature = "integration")))] pub(crate) mod test_utils; mod worker_connection_pool; mod worker_service; From 996364b0d9474ea33385f8311dbca9798c3e8ad4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Mon, 29 Jun 2026 14:37:05 -0700 Subject: [PATCH 05/15] Added an in-process worker transport that needs no gRPC. InProcessChannelResolver routes the three protocol methods straight to a co-located Worker, with no gRPC, no IPC, and no serialization round-trip: the reference implementation of the protocol for a co-located worker, and the first transport that exercises the abstraction with grpc off. Its end-to-end test (a distributed GROUP BY across tasks) runs under the no-gRPC CI job. Co-authored-by: Stu Hood --- src/lib.rs | 6 +- src/protocol/channel_resolver.rs | 3 + src/protocol/in_process.rs | 230 +++++++++++++++++++++++++++++++ src/protocol/mod.rs | 2 + 4 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 src/protocol/in_process.rs diff --git a/src/lib.rs b/src/lib.rs index d8d1f88fa..c64c7b2f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,9 +58,9 @@ pub use worker_resolver::{WorkerResolver, get_distributed_worker_resolver}; pub use protocol::{ ChannelResolver, CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, - GetWorkerInfoResponse, LoadInfo, ProducerHeadSpec, SetPlanRequest, TaskKey, TaskMetrics, - WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, WorkerToCoordinatorMsg, - get_distributed_channel_resolver, + GetWorkerInfoResponse, InProcessChannelResolver, LoadInfo, ProducerHeadSpec, SetPlanRequest, + TaskKey, TaskMetrics, WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, + WorkerToCoordinatorMsg, get_distributed_channel_resolver, }; pub use stage::{ DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, diff --git a/src/protocol/channel_resolver.rs b/src/protocol/channel_resolver.rs index e781aa901..05da30696 100644 --- a/src/protocol/channel_resolver.rs +++ b/src/protocol/channel_resolver.rs @@ -70,6 +70,9 @@ pub fn get_distributed_channel_resolver( #[cfg(not(feature = "grpc"))] { + // With `grpc` off there is no built-in default. A co-located deployment can register + // [`crate::InProcessChannelResolver`] (or another transport) via + // `with_distributed_channel_resolver`. panic!( "gRPC feature is not enabled, and no channel resolver was provided, so no default ChannelResolver can be provided" ); diff --git a/src/protocol/in_process.rs b/src/protocol/in_process.rs new file mode 100644 index 000000000..9a59085db --- /dev/null +++ b/src/protocol/in_process.rs @@ -0,0 +1,230 @@ +//! The reference implementation of the worker protocol for a co-located worker. +//! +//! It implements [`WorkerChannel`] by calling a [`Worker`] that lives in the same process, with no +//! gRPC, no IPC, and no networking. Its purpose is twofold: it lets the crate run distributed +//! queries with the `grpc` feature off (the protocol abstraction is only real if something other +//! than gRPC implements it), and it is the shape a custom co-located transport (for example a +//! shared-memory mesh spanning sibling processes) follows: implement [`ChannelResolver`] to hand +//! out a channel for a URL, then route the three protocol methods to wherever the worker runs. + +use crate::protocol::{ChannelResolver, WorkerChannel}; +use crate::{ + CoordinatorToWorkerMsg, DefaultSessionBuilder, DistributedExt, ExecuteTaskRequest, + GetWorkerInfoRequest, GetWorkerInfoResponse, MappedWorkerSessionBuilderExt, Worker, + WorkerSessionBuilder, WorkerToCoordinatorMsg, +}; +use async_trait::async_trait; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::{DataFusionError, Result, internal_datafusion_err}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use futures::stream::{BoxStream, StreamExt}; +use http::HeaderMap; +use std::sync::{Arc, Weak}; +use url::Url; + +/// A [`ChannelResolver`] backed by a single co-located [`Worker`]. +/// +/// Every URL resolves to that one worker: with no network there is nothing to dial, so the URLs a +/// [`crate::WorkerResolver`] hands out only label the tasks the planner routes. One worker holds the +/// state for every task, keyed by [`crate::TaskKey`], the same way the gRPC worker does when several +/// tasks of a query land on it. +#[derive(Clone)] +pub struct InProcessChannelResolver { + worker: Arc, +} + +impl InProcessChannelResolver { + /// Builds the co-located worker from `session_builder`, registering this resolver into the + /// worker's own per-query sessions so a worker reading a downstream stage stays in process + /// rather than falling back to the gRPC default. + pub fn from_session_builder( + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> Self { + // The worker and the resolver point at each other: the resolver runs tasks on the worker, + // and the worker resolves its own nested reads back through the resolver. A `Weak` on the + // worker's side breaks the cycle, so the returned `InProcessChannelResolver` owns the only + // strong reference and dropping it frees the worker. + let worker = Arc::new_cyclic(|weak: &Weak| { + let weak = weak.clone(); + Worker::from_session_builder(session_builder.map(move |builder| { + Ok(builder + .with_distributed_channel_resolver(WeakInProcessChannelResolver(weak.clone())) + .build()) + })) + }); + Self { worker } + } +} + +impl Default for InProcessChannelResolver { + fn default() -> Self { + Self::from_session_builder(DefaultSessionBuilder) + } +} + +#[async_trait] +impl ChannelResolver for InProcessChannelResolver { + async fn get_worker_client_for_url( + &self, + _url: &Url, + ) -> Result, DataFusionError> { + Ok(Box::new(InProcessWorkerChannel { + worker: Arc::clone(&self.worker), + })) + } +} + +/// The resolver a worker installs in its own sessions. It upgrades a [`Weak`] reference to the +/// co-located worker so a read of a downstream stage routes back in process instead of dialing out. +struct WeakInProcessChannelResolver(Weak); + +#[async_trait] +impl ChannelResolver for WeakInProcessChannelResolver { + async fn get_worker_client_for_url( + &self, + _url: &Url, + ) -> Result, DataFusionError> { + let worker = self + .0 + .upgrade() + .ok_or_else(|| internal_datafusion_err!("the in-process worker has been dropped"))?; + Ok(Box::new(InProcessWorkerChannel { worker })) + } +} + +/// A [`WorkerChannel`] that calls a co-located [`Worker`] directly. +struct InProcessWorkerChannel { + worker: Arc, +} + +#[async_trait] +impl WorkerChannel for InProcessWorkerChannel { + async fn coordinator_channel( + &mut self, + headers: HeaderMap, + c2w_stream: BoxStream<'static, CoordinatorToWorkerMsg>, + ) -> Result>> { + // The worker reads a fallible stream so a wire transport can surface its decode errors. + // Handing messages over in process has no such step, so each one is already an `Ok`. + self.worker + .coordinator_channel(headers, c2w_stream.map(Ok).boxed()) + .await + } + + async fn execute_task( + &mut self, + _headers: HeaderMap, + request: ExecuteTaskRequest, + _metrics: ExecutionPlanMetricsSet, + _task_ctx: &Arc, + ) -> Result>>> { + // Reading a partition runs the producer in place: the returned streams are the worker's own + // task output, so there is no IPC decode pass and no network metrics to record. The + // consumer's `task_ctx` is the consumer side's; the producer runs under the worker's own. + let (streams, _task_ctx) = self.worker.execute_task(request).await?; + Ok(streams.into_iter().map(|stream| stream.boxed()).collect()) + } + + async fn get_worker_info( + &mut self, + _request: GetWorkerInfoRequest, + ) -> Result { + Ok(GetWorkerInfoResponse { + version: self.worker.version().to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{SessionStateBuilderExt, WorkerResolver, display_plan_ascii}; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::collect; + use datafusion::prelude::{CsvReadOptions, SessionConfig, SessionContext}; + use std::io::Write; + + /// Hands out as many placeholder URLs as workers. With one co-located worker behind the + /// transport, these only label the tasks the planner routes; nothing is dialed. + struct InProcessWorkers(usize); + + impl WorkerResolver for InProcessWorkers { + fn get_urls(&self) -> Result, DataFusionError> { + (0..self.0) + .map(|i| Url::parse(&format!("http://worker-{i}"))) + .collect::>() + .map_err(|err| DataFusionError::External(Box::new(err))) + } + } + + /// Drives a real distributed query end to end through the in-process transport. With the `grpc` + /// feature off this is the only transport that can run it; `cargo check --no-default-features` + /// covers the no-gRPC compile. + #[tokio::test] + async fn in_process_transport_runs_a_distributed_query() -> Result<()> { + const N: usize = 4; + + // A file scan round-trips through `datafusion-proto`, so a worker can rebuild it from the + // serialized stage plan. An in-memory table would not, hence a CSV on disk. + let path = std::env::temp_dir().join(format!("dfd_in_process_{}.csv", std::process::id())); + let mut file = + std::fs::File::create(&path).map_err(|e| DataFusionError::External(Box::new(e)))?; + writeln!(file, "k,v").unwrap(); + for i in 0..200 { + writeln!(file, "{},{}", ["a", "b", "c", "d"][i % 4], i).unwrap(); + } + drop(file); + let path = path.to_str().unwrap().to_string(); + + let query = "SELECT k, COUNT(*) AS c FROM t GROUP BY k ORDER BY k"; + + // Single-node reference result. + let ctx = SessionContext::new_with_config(SessionConfig::new().with_target_partitions(N)); + ctx.register_csv("t", &path, CsvReadOptions::new()).await?; + let expected = collect( + ctx.sql(query).await?.create_physical_plan().await?, + ctx.task_ctx(), + ) + .await?; + let expected = pretty_format_batches(&expected)?.to_string(); + + // Distributed over the in-process transport. + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::new().with_target_partitions(N)) + .with_distributed_planner() + .with_distributed_worker_resolver(InProcessWorkers(N)) + .with_distributed_channel_resolver(InProcessChannelResolver::default()) + .with_distributed_file_scan_config_bytes_per_partition(1) + .unwrap() + .build(); + let ctx_distributed = SessionContext::from(state); + ctx_distributed + .register_csv("t", &path, CsvReadOptions::new()) + .await?; + + let physical = ctx_distributed + .sql(query) + .await? + .create_physical_plan() + .await?; + let rendered = display_plan_ascii(physical.as_ref(), false); + assert!( + rendered.contains("DistributedExec"), + "plan was not distributed:\n{rendered}" + ); + assert!( + rendered.contains("NetworkShuffleExec"), + "no shuffle boundary, so the transport never carried a cross-task stream:\n{rendered}" + ); + + let actual = collect(physical, ctx_distributed.task_ctx()).await?; + let actual = pretty_format_batches(&actual)?.to_string(); + + let _ = std::fs::remove_file(&path); + assert_eq!(actual, expected); + Ok(()) + } +} diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 024658766..3aee12733 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -6,10 +6,12 @@ mod channel_resolver; // worker, a shared-memory mesh) can speak the same wire shape without pulling in the whole gRPC // stack. pub(crate) mod generated; +mod in_process; mod worker_channel; pub(crate) use channel_resolver::set_distributed_channel_resolver; pub use channel_resolver::{ChannelResolver, get_distributed_channel_resolver}; +pub use in_process::InProcessChannelResolver; pub use worker_channel::{ CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, GetWorkerInfoResponse, From 5008211ebe9bf46934ade348d15227d66bae246a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 9 Jul 2026 16:02:52 -0700 Subject: [PATCH 06/15] Re-exposed the worker-metrics API an out-of-crate driver needs. A transport that returns worker metrics out-of-band, rather than over the coordinator_channel return stream, needs its driver to decode the frames and file them into the executed plan's store before the per-task EXPLAIN rewrite reads it: metrics_store() with a public insert, a no-gRPC decode_task_metrics, and the frame builders collect_plan_metrics_protos and set_received_time. The metrics codec moves out from behind grpc with them, and the gRPC client drops its private copy of the decode. The test pins the frame-to-store path. Co-authored-by: Stu Hood --- src/coordinator/distributed.rs | 7 +++ src/coordinator/metrics_store.rs | 4 +- src/coordinator/mod.rs | 2 +- src/lib.rs | 11 ++-- src/protocol/grpc/mod.rs | 1 - src/protocol/grpc/worker_client.rs | 21 +------ src/protocol/grpc/worker_service.rs | 2 +- src/protocol/{grpc => }/metrics_proto.rs | 70 ++++++++++++++++++++- src/protocol/mod.rs | 4 ++ src/work_unit_feed/mod.rs | 1 + src/work_unit_feed/remote_work_unit_feed.rs | 7 +++ src/worker/impl_coordinator_channel.rs | 22 +++++++ src/worker/mod.rs | 1 + 13 files changed, 126 insertions(+), 27 deletions(-) rename src/protocol/{grpc => }/metrics_proto.rs (95%) diff --git a/src/coordinator/distributed.rs b/src/coordinator/distributed.rs index cb126a884..a2cfa2ebd 100644 --- a/src/coordinator/distributed.rs +++ b/src/coordinator/distributed.rs @@ -64,6 +64,13 @@ impl DistributedExec { } } + /// The store where worker task metrics land at runtime, if metrics collection is enabled. + /// Exposed for a driver whose transport returns metrics out-of-band; it files decoded frames + /// here before the per-task EXPLAIN rewrite. + pub fn metrics_store(&self) -> Option> { + self.metrics_store.clone() + } + /// Enables task metrics collection from remote workers. pub fn with_metrics_collection(mut self, enabled: bool) -> Self { self.metrics_store = match enabled { diff --git a/src/coordinator/metrics_store.rs b/src/coordinator/metrics_store.rs index ed55db2c3..b38d62a03 100644 --- a/src/coordinator/metrics_store.rs +++ b/src/coordinator/metrics_store.rs @@ -17,7 +17,9 @@ impl MetricsStore { Self { tx, rx } } - pub(crate) fn insert(&self, key: TaskKey, metrics: TaskMetrics) { + // Public for a driver whose transport returns worker metrics out-of-band; it files the + // decoded frames here before the per-task EXPLAIN rewrite reads them. + pub fn insert(&self, key: TaskKey, metrics: TaskMetrics) { self.tx.send_modify(|map| { map.insert(key, metrics); }); diff --git a/src/coordinator/mod.rs b/src/coordinator/mod.rs index c1a8a8dd2..be069485e 100644 --- a/src/coordinator/mod.rs +++ b/src/coordinator/mod.rs @@ -6,4 +6,4 @@ mod prepare_static_plan; mod query_coordinator; pub use distributed::DistributedExec; -pub(crate) use metrics_store::MetricsStore; +pub use metrics_store::MetricsStore; diff --git a/src/lib.rs b/src/lib.rs index c64c7b2f3..032e56bd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,9 @@ mod worker_resolver; #[cfg(feature = "grpc")] pub use arrow_ipc::CompressionType; -pub use coordinator::DistributedExec; +// `MetricsStore` is public for a driver whose transport returns worker metrics out-of-band; it +// files the decoded frames into the executed plan's store before the per-task EXPLAIN rewrite. +pub use coordinator::{DistributedExec, MetricsStore}; pub use distributed_ext::{DistributedExt, DistributedGetterExt}; pub use distributed_planner::{ DistributedConfig, NetworkBoundary, NetworkBoundaryExt, SessionStateBuilderExt, @@ -60,17 +62,18 @@ pub use protocol::{ ChannelResolver, CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, GetWorkerInfoResponse, InProcessChannelResolver, LoadInfo, ProducerHeadSpec, SetPlanRequest, TaskKey, TaskMetrics, WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, - WorkerToCoordinatorMsg, get_distributed_channel_resolver, + WorkerToCoordinatorMsg, decode_task_metrics, get_distributed_channel_resolver, }; pub use stage::{ DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, }; pub use work_unit_feed::{ - DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, WorkUnitFeedProvider, + DistributedWorkUnitFeedContext, WorkUnit, WorkUnitFeed, WorkUnitFeedProto, + WorkUnitFeedProvider, set_received_time, }; pub use worker::{ DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, TaskData, - Worker, WorkerQueryContext, WorkerSessionBuilder, + Worker, WorkerQueryContext, WorkerSessionBuilder, collect_plan_metrics_protos, }; #[cfg(all(feature = "grpc", any(feature = "integration", test)))] diff --git a/src/protocol/grpc/mod.rs b/src/protocol/grpc/mod.rs index fc367585f..cc2f65760 100644 --- a/src/protocol/grpc/mod.rs +++ b/src/protocol/grpc/mod.rs @@ -1,6 +1,5 @@ mod channel_resolver; mod errors; -mod metrics_proto; mod observability; mod on_drop_stream; mod spawn_select_all; diff --git a/src/protocol/grpc/worker_client.rs b/src/protocol/grpc/worker_client.rs index d342eebe1..f8f31ad4e 100644 --- a/src/protocol/grpc/worker_client.rs +++ b/src/protocol/grpc/worker_client.rs @@ -1,16 +1,16 @@ use super::channel_resolver::BoxCloneSyncChannel; use super::errors::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; -use super::metrics_proto::metrics_set_proto_to_df; use crate::common::serialize_uuid; use crate::grpc::on_drop_stream::on_drop_stream; use crate::protocol::generated::worker as pb; use crate::protocol::generated::worker::FlightAppMetadata; +use crate::protocol::metrics_proto::decode_task_metrics; use crate::{ BytesMetricExt, CoordinatorToWorkerMsg, DistributedConfig, ExecuteTaskRequest, FirstLatencyMetric, GetWorkerInfoRequest, GetWorkerInfoResponse, LatencyMetricExt, LoadInfo, MaxLatencyMetric, MinLatencyMetric, P50LatencyMetric, P95LatencyMetric, ProducerHeadSpec, - SetPlanRequest, TaskKey, TaskMetrics, WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, - WorkerChannel, WorkerToCoordinatorMsg, + SetPlanRequest, TaskKey, WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, + WorkerToCoordinatorMsg, }; use arrow_flight::FlightData; use arrow_flight::decode::FlightRecordBatchStream; @@ -491,21 +491,6 @@ fn decode_worker_to_coordinator_msg( ) } -fn decode_task_metrics(task_metrics: pb::TaskMetrics) -> Result { - Ok(TaskMetrics { - pre_order_plan_metrics: task_metrics - .pre_order_plan_metrics - .into_iter() - .map(|metrics_set| metrics_set_proto_to_df(&metrics_set)) - .collect::>()?, - task_metrics: metrics_set_proto_to_df( - &task_metrics - .task_metrics - .ok_or_else(|| missing("task_metrics"))?, - )?, - }) -} - fn decode_load_info(load_info: pb::LoadInfo) -> LoadInfo { LoadInfo { partition: load_info.partition as usize, diff --git a/src/protocol/grpc/worker_service.rs b/src/protocol/grpc/worker_service.rs index 831865a9b..6282d2ad6 100644 --- a/src/protocol/grpc/worker_service.rs +++ b/src/protocol/grpc/worker_service.rs @@ -1,7 +1,7 @@ use super::errors::{datafusion_error_to_tonic_status, map_status_to_datafusion_error}; -use super::metrics_proto::df_metrics_set_to_proto; use super::spawn_select_all::spawn_select_all; use crate::protocol::generated::worker as pb; +use crate::protocol::metrics_proto::df_metrics_set_to_proto; use crate::common::{deserialize_uuid, now_ns}; use crate::protocol::ProducerHeadSpec; diff --git a/src/protocol/grpc/metrics_proto.rs b/src/protocol/metrics_proto.rs similarity index 95% rename from src/protocol/grpc/metrics_proto.rs rename to src/protocol/metrics_proto.rs index 9dfc145ed..22127672e 100644 --- a/src/protocol/grpc/metrics_proto.rs +++ b/src/protocol/metrics_proto.rs @@ -1,4 +1,4 @@ -use crate::protocol::generated::worker as pb; +use super::generated::worker as pb; use chrono::DateTime; use datafusion::common::internal_err; use datafusion::error::DataFusionError; @@ -55,6 +55,27 @@ pub fn metrics_set_proto_to_df( Ok(metrics_set) } +/// Decode a wire [`pb::TaskMetrics`] into the in-memory [`crate::TaskMetrics`]. The no-gRPC push +/// embedder receives metric frames as proto and files them into the plain metrics store, so it +/// needs the decode direction without the gRPC client. +pub fn decode_task_metrics( + task_metrics: pb::TaskMetrics, +) -> Result { + Ok(crate::TaskMetrics { + pre_order_plan_metrics: task_metrics + .pre_order_plan_metrics + .iter() + .map(metrics_set_proto_to_df) + .collect::>()?, + task_metrics: metrics_set_proto_to_df( + task_metrics + .task_metrics + .as_ref() + .ok_or_else(|| DataFusionError::Internal("Missing field 'task_metrics'".into()))?, + )?, + }) +} + /// Custom metrics are not supported in proto conversion. const CUSTOM_METRICS_NOT_SUPPORTED: &str = "custom metrics are not supported in metrics proto conversion"; @@ -1332,3 +1353,50 @@ mod tests { } } } + +#[cfg(test)] +mod decode_tests { + use super::*; + use crate::TaskKey; + use crate::coordinator::MetricsStore; + use datafusion::physical_plan::metrics::{Count, Metric, MetricValue, MetricsSet}; + + /// Pins the driver-facing decode path: a proto metrics frame (the wire form a transport + /// delivers out-of-band) decodes to the plain form and files into the store the per-task + /// EXPLAIN rewrite reads. + #[test] + fn a_metrics_frame_decodes_and_files_into_the_store() { + let count = Count::new(); + count.add(42); + let mut set = MetricsSet::new(); + set.push(Arc::new(Metric::new( + MetricValue::OutputRows(count), + Some(0), + ))); + let node_metrics = df_metrics_set_to_proto(&set).expect("encode"); + + let decoded = decode_task_metrics(pb::TaskMetrics { + pre_order_plan_metrics: vec![node_metrics], + task_metrics: Some(pb::MetricsSet::default()), + }) + .expect("decode"); + assert_eq!(decoded.pre_order_plan_metrics.len(), 1); + assert_eq!( + decoded.pre_order_plan_metrics[0].output_rows(), + Some(42), + "the counter survives the wire round trip" + ); + + let store = MetricsStore::new(); + let key = TaskKey { + query_id: uuid::Uuid::new_v4(), + stage_id: 1, + task_number: 0, + }; + store.insert(key, decoded); + assert!( + store.get(&key).is_some(), + "the driver-filed entry is readable" + ); + } +} diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 3aee12733..aa8ec3181 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -7,11 +7,15 @@ mod channel_resolver; // stack. pub(crate) mod generated; mod in_process; +// The metrics codec sits off gRPC for the same reason as the message types: a transport that +// delivers metrics out-of-band decodes the same frames the gRPC client does. +pub(crate) mod metrics_proto; mod worker_channel; pub(crate) use channel_resolver::set_distributed_channel_resolver; pub use channel_resolver::{ChannelResolver, get_distributed_channel_resolver}; pub use in_process::InProcessChannelResolver; +pub use metrics_proto::decode_task_metrics; pub use worker_channel::{ CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, GetWorkerInfoResponse, diff --git a/src/work_unit_feed/mod.rs b/src/work_unit_feed/mod.rs index 1bd55ebfb..ec74f52ce 100644 --- a/src/work_unit_feed/mod.rs +++ b/src/work_unit_feed/mod.rs @@ -11,6 +11,7 @@ pub(crate) use remote_work_unit_feed::{ }; pub(crate) use work_unit_feed_registry::{WorkUnitFeedRegistry, set_distributed_work_unit_feed}; +pub use remote_work_unit_feed::set_received_time; pub use work_unit::WorkUnit; pub use work_unit_feed::{WorkUnitFeed, WorkUnitFeedProto}; pub use work_unit_feed_provider::{DistributedWorkUnitFeedContext, WorkUnitFeedProvider}; diff --git a/src/work_unit_feed/remote_work_unit_feed.rs b/src/work_unit_feed/remote_work_unit_feed.rs index 18c51ffe4..78ecc42aa 100644 --- a/src/work_unit_feed/remote_work_unit_feed.rs +++ b/src/work_unit_feed/remote_work_unit_feed.rs @@ -99,6 +99,13 @@ pub(crate) fn set_work_unit_received_time( msg } +/// Stamps the receive time on a bare proto work unit. A transport that hands units across without +/// going through [set_work_unit_received_time] stamps each one as it crosses into the worker, so the +/// worker-side latency math has a delivery timestamp; a missing stamp reads as zero latency. +pub fn set_received_time(work_unit: &mut crate::proto::WorkUnit) { + work_unit.received_timestamp_unix_nanos = now_ns(); +} + /// Remove implementation of a [WorkUnitFeedProvider] that pulls [crate::WorkUnitMsg]s coming over /// the wire from a [RemoteWorkUnitFeedRegistry]. /// diff --git a/src/worker/impl_coordinator_channel.rs b/src/worker/impl_coordinator_channel.rs index 020bce948..c6efc14c8 100644 --- a/src/worker/impl_coordinator_channel.rs +++ b/src/worker/impl_coordinator_channel.rs @@ -1,6 +1,7 @@ use crate::common::TreeNodeExt; use crate::events::{WorkerPlanRewriteEvent, WorkerPlanRewriteHandlers}; use crate::execution_plans::SamplerExec; +use crate::protocol::metrics_proto::df_metrics_set_to_proto; use crate::work_unit_feed::{RemoteWorkUnitFeedRegistry, set_work_unit_received_time}; use crate::worker::LocalWorkerContext; use crate::worker::task_data::TaskDataMetrics; @@ -232,3 +233,24 @@ fn send_metrics_via_channel( task_metrics: task_data_metrics.to_metrics_set(), }); } + +/// Collects the per-node metrics of `plan` in pre-order traversal order, encoded as protos. A +/// push transport reuses this for its metrics frame so the per-node shape matches what the +/// coordinator channel reports. A node without metrics, or one whose metrics fail to encode, +/// contributes an empty set in place: the consumer matches entries to plan nodes by pre-order +/// position, so dropping an entry would mis-attribute every set after it. +pub fn collect_plan_metrics_protos( + plan: &Arc, + dt_ctx: DistributedTaskContext, +) -> Vec { + let mut pre_order_plan_metrics = vec![]; + let _ = plan.apply_with_dt_ctx(dt_ctx, |node, _| { + pre_order_plan_metrics.push( + node.metrics() + .and_then(|m| df_metrics_set_to_proto(&m).ok()) + .unwrap_or_default(), + ); + Ok(TreeNodeRecursion::Continue) + }); + pre_order_plan_metrics +} diff --git a/src/worker/mod.rs b/src/worker/mod.rs index 030ac357f..70dc64676 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod test_utils; mod worker_connection_pool; mod worker_service; +pub use impl_coordinator_channel::collect_plan_metrics_protos; pub(crate) use single_write_multi_read::SingleWriteMultiRead; pub(crate) use worker_connection_pool::WorkerConnectionPool; From 450c8862daa2b5c3e99465ab37fabaedd35260a3 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 1 Jul 2026 11:34:42 -0700 Subject: [PATCH 07/15] Add a shared-memory transport. In support of that: * Added a DispatchPlanSource hook for coordinator-sourced plans. * Exposed produce-side partition routing off the network boundaries. * Routed dispatched plans to workers over the shm mesh. * fix(shm): stream oversized frames through the ring (#51) --- Cargo.lock | 24 + Cargo.toml | 2 + src/coordinator/query_coordinator.rs | 20 +- src/dispatch_plan_source.rs | 241 ++ src/distributed_ext.rs | 32 +- src/distributed_planner/mod.rs | 2 +- src/distributed_planner/network_boundary.rs | 130 +- src/execution_plans/network_coalesce.rs | 10 + src/lib.rs | 11 +- src/shm/dsm.rs | 544 +++ src/shm/in_process.rs | 1109 ++++++ src/shm/mesh.rs | 379 ++ src/shm/mod.rs | 81 + src/shm/mpsc_ring.rs | 1560 ++++++++ src/shm/runtime.rs | 551 +++ src/shm/self_hosted.rs | 1274 +++++++ src/shm/setup.rs | 373 ++ src/shm/sink.rs | 49 + src/shm/transport.rs | 3539 +++++++++++++++++++ src/work_unit_feed/mod.rs | 1 + 20 files changed, 9921 insertions(+), 11 deletions(-) create mode 100644 src/dispatch_plan_source.rs create mode 100644 src/shm/dsm.rs create mode 100644 src/shm/in_process.rs create mode 100644 src/shm/mesh.rs create mode 100644 src/shm/mod.rs create mode 100644 src/shm/mpsc_ring.rs create mode 100644 src/shm/runtime.rs create mode 100644 src/shm/self_hosted.rs create mode 100644 src/shm/setup.rs create mode 100644 src/shm/sink.rs create mode 100644 src/shm/transport.rs diff --git a/Cargo.lock b/Cargo.lock index 346b3aed8..c2df9879f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -485,6 +485,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -2101,6 +2123,7 @@ dependencies = [ "arrow-flight", "arrow-ipc", "arrow-select", + "async-stream", "async-trait", "bincode", "bytes", @@ -2115,6 +2138,7 @@ dependencies = [ "hyper-util", "insta", "itertools 0.14.0", + "log", "moka", "num-traits", "object_store", diff --git a/Cargo.toml b/Cargo.toml index c38bc079c..7a1deefaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ datafusion = { workspace = true, features = [ ] } datafusion-proto = { workspace = true } async-trait = "0.1.89" +async-stream = "0.3" +log = "0.4" tokio = { version = "1.48", features = ["full"] } http = "1.3.1" itertools = "0.14.0" diff --git a/src/coordinator/query_coordinator.rs b/src/coordinator/query_coordinator.rs index 3acced181..4e31af209 100644 --- a/src/coordinator/query_coordinator.rs +++ b/src/coordinator/query_coordinator.rs @@ -12,7 +12,7 @@ use crate::{ BytesCounterMetric, BytesMetricExt, CoordinatorToWorkerMsg, DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, DistributedCodec, DistributedTaskContext, DistributedWorkUnitFeedContext, LoadInfo, SetPlanRequest, TaskKey, WorkUnitFeedDeclaration, - WorkerToCoordinatorMsg, get_distributed_channel_resolver, + WorkerToCoordinatorMsg, get_distributed_channel_resolver, get_distributed_dispatch_plan_source, }; use datafusion::common::DataFusionError; use datafusion::common::instant::Instant; @@ -140,19 +140,27 @@ impl<'a> StageCoordinator<'a> { UnboundedReceiver, )> { let session_config = self.task_ctx.session_config(); - let codec = DistributedCodec::new_combined_with_user(session_config); let (specialized, work_unit_feed_declarations) = self.task_specialized_plan(task_i)?; - let plan_proto = - PhysicalPlanNode::try_from_physical_plan(specialized, &codec)?.encode_to_vec(); - let plan_size = plan_proto.len(); - + // An embedder can serialize the dispatch bytes for this stage itself (e.g. with a codec + // the config's extension point cannot express) instead of the coordinator encoding the + // plan. Either way the bytes describe `specialized`, the ready-to-run per-task plan. let task_key = TaskKey { query_id: self.query_id, stage_id: self.stage_id, task_number: task_i, }; + let plan_proto = match get_distributed_dispatch_plan_source(session_config) + .and_then(|source| source.dispatch_plan_proto(&task_key, &specialized)) + { + Some(bytes) => bytes?, + None => { + let codec = DistributedCodec::new_combined_with_user(session_config); + PhysicalPlanNode::try_from_physical_plan(specialized, &codec)?.encode_to_vec() + } + }; + let plan_size = plan_proto.len(); let msg = CoordinatorToWorkerMsg::SetPlanRequest(SetPlanRequest { task_key, diff --git a/src/dispatch_plan_source.rs b/src/dispatch_plan_source.rs new file mode 100644 index 000000000..d7f849944 --- /dev/null +++ b/src/dispatch_plan_source.rs @@ -0,0 +1,241 @@ +use crate::TaskKey; +use datafusion::common::Result; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionConfig; +use std::sync::Arc; + +/// Serializes the stage subplan the coordinator dispatches, instead of the coordinator encoding +/// it with its own codec. +/// +/// An embedder registers one via +/// [`crate::DistributedExt::with_distributed_dispatch_plan_source`] when the coordinator's codec +/// cannot represent its plan nodes, or when its serialization needs embedder-side handling the +/// codec extension point cannot express (the shm embedder's UDF definitions, for example). The +/// coordinator hands over `specialized`, the same ready-to-run per-task plan it would encode: +/// task-specialized, with nested stages already converted to `Remote`, so a worker executes the +/// decoded bytes as-is. `task` carries the query id, so a source registered on a session that +/// runs concurrent queries can tell them apart. +/// +/// Returning `None` for a task lets the coordinator fall back to encoding the plan itself, so a +/// source that only overrides some stages stays correct. +pub trait DispatchPlanSource: Send + Sync { + fn dispatch_plan_proto( + &self, + task: &TaskKey, + specialized: &Arc, + ) -> Option>>; +} + +#[derive(Clone)] +pub(crate) struct DispatchPlanSourceExtension(pub(crate) Arc); + +pub(crate) fn set_distributed_dispatch_plan_source( + cfg: &mut SessionConfig, + source: impl DispatchPlanSource + 'static, +) { + set_distributed_dispatch_plan_source_arc(cfg, Arc::new(source)) +} + +pub(crate) fn set_distributed_dispatch_plan_source_arc( + cfg: &mut SessionConfig, + source: Arc, +) { + cfg.set_extension(Arc::new(DispatchPlanSourceExtension(source))); +} + +/// Returns the [`DispatchPlanSource`] registered on this config, if any. +pub fn get_distributed_dispatch_plan_source( + cfg: &SessionConfig, +) -> Option> { + cfg.get_extension::() + .map(|ext| Arc::clone(&ext.0)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::InProcessChannelResolver; + use crate::{ + DistributedExt, NetworkBoundaryExt, SessionStateBuilderExt, Stage, WorkerResolver, + }; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::common::{DataFusionError, Result}; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion::prelude::{CsvReadOptions, SessionConfig, SessionContext}; + use datafusion_proto::physical_plan::AsExecutionPlan; + use datafusion_proto::protobuf::PhysicalPlanNode; + use prost::Message; + use std::io::Write; + use std::sync::Mutex; + use url::Url; + + struct Workers(usize); + + impl WorkerResolver for Workers { + fn get_urls(&self) -> Result> { + (0..self.0) + .map(|i| Url::parse(&format!("http://worker-{i}"))) + .collect::>() + .map_err(|err| DataFusionError::External(Box::new(err))) + } + } + + type Calls = Arc)>>>; + + /// Records what the coordinator hands over and declines, so the coordinator's own encode + /// still runs and the query is unaffected by the recording. + #[derive(Default)] + struct RecordingSource { + calls: Calls, + } + + impl DispatchPlanSource for RecordingSource { + fn dispatch_plan_proto( + &self, + task: &TaskKey, + specialized: &Arc, + ) -> Option>> { + self.calls + .lock() + .unwrap() + .push((*task, Arc::clone(specialized))); + None + } + } + + const QUERY: &str = "SELECT k, COUNT(*) AS c FROM t GROUP BY k ORDER BY k"; + + async fn distributed_ctx( + name: &str, + source: impl DispatchPlanSource + 'static, + ) -> Result<(SessionContext, std::path::PathBuf)> { + let path = std::env::temp_dir().join(format!("dfd_{name}_{}.csv", std::process::id())); + let mut file = + std::fs::File::create(&path).map_err(|e| DataFusionError::External(Box::new(e)))?; + writeln!(file, "k,v").unwrap(); + for i in 0..200 { + writeln!(file, "{},{}", ["a", "b", "c", "d"][i % 4], i).unwrap(); + } + drop(file); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::new().with_target_partitions(4)) + .with_distributed_planner() + .with_distributed_worker_resolver(Workers(4)) + .with_distributed_channel_resolver(InProcessChannelResolver::default()) + .with_distributed_dispatch_plan_source(source) + .with_distributed_file_scan_config_bytes_per_partition(1) + .unwrap() + .build(); + let ctx = SessionContext::from(state); + ctx.register_csv("t", path.to_str().unwrap(), CsvReadOptions::new()) + .await?; + Ok((ctx, path)) + } + + /// The contract a serializing source relies on: it is consulted once per dispatched task, + /// and the plan it is handed is ready to run, with nested stages already converted to + /// `Remote`. + #[tokio::test] + async fn source_is_consulted_with_the_ready_to_run_plan() -> Result<()> { + let source = RecordingSource::default(); + let calls = Arc::clone(&source.calls); + let (ctx, path) = distributed_ctx("recording", source).await?; + + let physical = ctx.sql(QUERY).await?.create_physical_plan().await?; + collect(physical, ctx.task_ctx()).await?; + + let calls = calls.lock().unwrap(); + assert!( + !calls.is_empty(), + "the coordinator never consulted the source" + ); + let mut seen: datafusion::common::HashSet = Default::default(); + for (task, specialized) in calls.iter() { + assert!(seen.insert(*task), "consulted twice for {task:?}"); + specialized.apply(|node| { + if let Some(nb) = node.as_ref().as_network_boundary() { + assert!( + matches!(nb.input_stage(), Stage::Remote(_)), + "{task:?} carries a Local nested stage; the handed-over plan must be \ + ready to run" + ); + } + Ok(TreeNodeRecursion::Continue) + })?; + } + std::fs::remove_file(&path).ok(); + Ok(()) + } + + /// Serializes with the same codec the coordinator's fallback uses, so the worker decodes + /// source-provided bytes exactly as it decodes coordinator-encoded ones. + struct SerializingSource; + + impl DispatchPlanSource for SerializingSource { + fn dispatch_plan_proto( + &self, + _task: &TaskKey, + specialized: &Arc, + ) -> Option>> { + let codec = crate::DistributedCodec::new_combined_with_user(&SessionConfig::new()); + Some( + PhysicalPlanNode::try_from_physical_plan(Arc::clone(specialized), &codec) + .map(|node| node.encode_to_vec()), + ) + } + } + + /// The bytes the source returns are what the workers run. + #[tokio::test] + async fn source_provided_bytes_run_the_query() -> Result<()> { + let (ctx, path) = distributed_ctx("serializing", SerializingSource).await?; + let got = ctx.sql(QUERY).await?.collect().await?; + let got = datafusion::arrow::util::pretty::pretty_format_batches(&got)?.to_string(); + + let serial = SessionContext::new(); + serial + .register_csv("t", path.to_str().unwrap(), CsvReadOptions::new()) + .await?; + let expected = serial.sql(QUERY).await?.collect().await?; + let expected = + datafusion::arrow::util::pretty::pretty_format_batches(&expected)?.to_string(); + + assert_eq!(got, expected, "source-encoded dispatch != serial"); + std::fs::remove_file(&path).ok(); + Ok(()) + } + + struct FailingSource; + + impl DispatchPlanSource for FailingSource { + fn dispatch_plan_proto( + &self, + _task: &TaskKey, + _specialized: &Arc, + ) -> Option>> { + Some(Err(DataFusionError::Internal( + "the embedder could not serialize this stage".into(), + ))) + } + } + + /// A source failure fails the dispatch instead of falling back to bytes the embedder said it + /// could not produce. + #[tokio::test] + async fn source_error_fails_the_query() -> Result<()> { + let (ctx, path) = distributed_ctx("failing", FailingSource).await?; + let result = ctx.sql(QUERY).await?.collect().await; + let err = result + .expect_err("a failing source must fail the query") + .to_string(); + assert!( + err.contains("could not serialize"), + "unexpected error: {err}" + ); + std::fs::remove_file(&path).ok(); + Ok(()) + } +} diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 5089038ef..93423c25f 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -2,6 +2,7 @@ use crate::codec::{set_distributed_user_codec, set_distributed_user_codec_arc}; use crate::config_extension_ext::{ set_distributed_option_extension, set_distributed_option_extension_from_headers, }; +use crate::dispatch_plan_source::set_distributed_dispatch_plan_source; use crate::events::{ DesiredTaskCountHandler, DesiredTaskCountHandlers, RouteTasksHandler, RouteTasksHandlers, ScaleUpLeafNodeHandler, ScaleUpLeafNodeHandlers, WorkerPlanRewriteHandler, @@ -12,8 +13,8 @@ use crate::protocol::set_distributed_channel_resolver; use crate::work_unit_feed::set_distributed_work_unit_feed; use crate::worker_resolver::set_distributed_worker_resolver; use crate::{ - ChannelResolver, DistributedConfig, LocalWorkerContext, WorkUnitFeed, WorkUnitFeedProvider, - WorkerResolver, get_distributed_worker_resolver, + ChannelResolver, DispatchPlanSource, DistributedConfig, LocalWorkerContext, WorkUnitFeed, + WorkUnitFeedProvider, WorkerResolver, get_distributed_worker_resolver, }; use datafusion::common::DataFusionError; use datafusion::config::ConfigExtension; @@ -282,6 +283,15 @@ pub trait DistributedExt: Sized { resolver: T, ); + /// Registers a [DispatchPlanSource] the coordinator consults for each stage's dispatch bytes + /// instead of encoding the plan it holds. See [DispatchPlanSource] for when this is needed. + fn with_distributed_dispatch_plan_source( + self, + source: T, + ) -> Self; + + /// Same as [DistributedExt::with_distributed_dispatch_plan_source] but with an in-place mutation. + fn set_distributed_dispatch_plan_source(&mut self, source: T); /// Sets the number of bytes each partition in a stage with a FileScanConfig node is /// expected to scan. A task runs `target_partitions` partitions, so the task count is /// roughly `total_scan_bytes / bytes_per_partition / target_partitions` (capped at the @@ -779,6 +789,9 @@ impl DistributedExt for SessionConfig { set_distributed_channel_resolver(self, resolver); } + fn set_distributed_dispatch_plan_source(&mut self, source: T) { + set_distributed_dispatch_plan_source(self, source); + } fn set_distributed_file_scan_config_bytes_per_partition( &mut self, bytes_per_partition: usize, @@ -945,6 +958,9 @@ impl DistributedExt for SessionConfig { #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + #[call(set_distributed_dispatch_plan_source)] + #[expr($;self)] + fn with_distributed_dispatch_plan_source(mut self, source: T) -> Self; #[call(set_distributed_file_scan_config_bytes_per_partition)] #[expr($?;Ok(self))] fn with_distributed_file_scan_config_bytes_per_partition(mut self, bytes_per_partition: usize) -> Result; @@ -1068,6 +1084,10 @@ impl DistributedExt for SessionStateBuilder { #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + fn set_distributed_dispatch_plan_source(&mut self, source: T); + #[call(set_distributed_dispatch_plan_source)] + #[expr($;self)] + fn with_distributed_dispatch_plan_source(mut self, source: T) -> Self; fn set_distributed_file_scan_config_bytes_per_partition(&mut self, bytes_per_partition: usize) -> Result<(), DataFusionError>; #[call(set_distributed_file_scan_config_bytes_per_partition)] #[expr($?;Ok(self))] @@ -1218,6 +1238,10 @@ impl DistributedExt for SessionState { #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + fn set_distributed_dispatch_plan_source(&mut self, source: T); + #[call(set_distributed_dispatch_plan_source)] + #[expr($;self)] + fn with_distributed_dispatch_plan_source(mut self, source: T) -> Self; fn set_distributed_file_scan_config_bytes_per_partition(&mut self, bytes_per_partition: usize) -> Result<(), DataFusionError>; #[call(set_distributed_file_scan_config_bytes_per_partition)] #[expr($?;Ok(self))] @@ -1361,6 +1385,10 @@ impl DistributedExt for SessionContext { #[expr($;self)] fn with_distributed_channel_resolver(self, resolver: T) -> Self; + fn set_distributed_dispatch_plan_source(&mut self, source: T); + #[call(set_distributed_dispatch_plan_source)] + #[expr($;self)] + fn with_distributed_dispatch_plan_source(self, source: T) -> Self; fn set_distributed_file_scan_config_bytes_per_partition(&mut self, bytes_per_partition: usize) -> Result<(), DataFusionError>; #[call(set_distributed_file_scan_config_bytes_per_partition)] #[expr($?;Ok(self))] diff --git a/src/distributed_planner/mod.rs b/src/distributed_planner/mod.rs index 5506331b2..dd8b31084 100644 --- a/src/distributed_planner/mod.rs +++ b/src/distributed_planner/mod.rs @@ -16,6 +16,6 @@ pub(crate) use inject_network_boundaries::{ InjectNetworkBoundaryContext, NetworkBoundaryBuilderResult, inject_network_boundaries, }; pub(crate) use network_boundary::ProducerHead; -pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt}; +pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt, PartitionRoute}; pub use session_state_builder_ext::SessionStateBuilderExt; pub(crate) use statistics::calculate_cost; diff --git a/src/distributed_planner/network_boundary.rs b/src/distributed_planner/network_boundary.rs index 3b7bd3b1f..05a00ad6c 100644 --- a/src/distributed_planner/network_boundary.rs +++ b/src/distributed_planner/network_boundary.rs @@ -5,7 +5,7 @@ use crate::{ Stage, }; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::Result; +use datafusion::common::{Result, internal_err}; use datafusion::execution::TaskContext; use datafusion::physical_expr::Partitioning; use datafusion::physical_plan::repartition::RepartitionExec; @@ -19,6 +19,14 @@ use datafusion_proto::protobuf::proto_error; use prost::Message; use std::sync::Arc; +/// Where a producer's output partition should be sent: which consumer task, and the local partition +/// index within that task's slice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PartitionRoute { + pub consumer_task: usize, + pub consumer_partition: usize, +} + /// This trait represents a node that introduces the necessity of a network boundary in the plan. /// The distributed planner, upon stepping into one of these, will break the plan and build a stage /// out of it. @@ -36,6 +44,31 @@ pub trait NetworkBoundary: ExecutionPlan { /// implementation have. This information is used during planning an executing for ensuring /// the head of a stage has the appropriate shape for consumption. fn producer_head(&self, consumer_tasks: usize) -> ProducerHead; + + /// Maps a producer output partition to the consumer task and the local partition within that + /// task that reads it, for the sliced layout shuffle and broadcast reads use + /// (`global = P_c * consumer_task + local`, where `P_c` is the boundary's own per-task output + /// partition count). A pull-based transport never needs this: its consumers compute their own + /// slice inside the boundary's `execute`. A push-based transport places every produced + /// partition before any consumer asks, so it reads the layout here instead of re-deriving it + /// and drifting when the layout changes. + /// + /// Boundaries whose consumers do not read that layout must override this with an error; the + /// default would silently misroute them. A zero-partition boundary is a planner bug, so it + /// errors instead of routing everything to task `0`. + fn route_partition(&self, output_partition: usize) -> Result { + let p_c = self.properties().partitioning.partition_count(); + if p_c == 0 { + return internal_err!( + "cannot route output partition {output_partition}: the boundary reports 0 \ + partitions per consumer task" + ); + } + Ok(PartitionRoute { + consumer_task: output_partition / p_c, + consumer_partition: output_partition % p_c, + }) + } } /// Defines what shape should the head node of a stage have upon getting executed. Depending @@ -155,3 +188,98 @@ impl ProducerHead { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::InProcessChannelResolver; + use crate::{DistributedExt, NetworkBoundaryExt, SessionStateBuilderExt, WorkerResolver}; + use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion::error::DataFusionError; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::{CsvReadOptions, SessionConfig, SessionContext}; + use std::io::Write; + use url::Url; + + struct Workers(usize); + + impl WorkerResolver for Workers { + fn get_urls(&self) -> Result> { + (0..self.0) + .map(|i| Url::parse(&format!("http://worker-{i}"))) + .collect::>() + .map_err(|err| DataFusionError::External(Box::new(err))) + } + } + + /// Pins the sliced routing (`global = P_c * consumer_task + local`) on boundaries the + /// planner actually built, with `P_c` read off the boundary's own properties, so an override + /// or a change in what `properties()` reports fails here first. The data-level guarantee + /// that the slicing matches what consumers read comes from the in-process transport's + /// end-to-end test. `NetworkCoalesceExec` must refuse to route: its consumers read whole + /// per-producer-task groups, and the sliced formula would misroute them. + #[tokio::test] + async fn route_partition_matches_the_consumer_slicing() -> Result<()> { + let path = std::env::temp_dir().join(format!("dfd_routing_{}.csv", std::process::id())); + let mut file = + std::fs::File::create(&path).map_err(|e| DataFusionError::External(Box::new(e)))?; + writeln!(file, "k,v").unwrap(); + for i in 0..200 { + writeln!(file, "{},{}", ["a", "b", "c", "d"][i % 4], i).unwrap(); + } + drop(file); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::new().with_target_partitions(4)) + .with_distributed_planner() + .with_distributed_worker_resolver(Workers(4)) + .with_distributed_channel_resolver(InProcessChannelResolver::default()) + .with_distributed_file_scan_config_bytes_per_partition(1) + .unwrap() + .build(); + let ctx = SessionContext::from(state); + ctx.register_csv("t", path.to_str().unwrap(), CsvReadOptions::new()) + .await?; + let physical = ctx + .sql("SELECT k, COUNT(*) AS c FROM t GROUP BY k ORDER BY k") + .await? + .create_physical_plan() + .await?; + + let mut sliced = 0usize; + let mut grouped = 0usize; + physical.apply(|node| { + let Some(nb) = node.as_ref().as_network_boundary() else { + return Ok(TreeNodeRecursion::Continue); + }; + if node + .as_ref() + .downcast_ref::() + .is_some() + { + assert!( + nb.route_partition(0).is_err(), + "a per-task-group boundary must refuse the sliced routing" + ); + grouped += 1; + return Ok(TreeNodeRecursion::Continue); + } + let p_c = nb.properties().partitioning.partition_count(); + assert!(p_c > 0); + for consumer_task in 0..3 { + for local in 0..p_c { + let route = nb.route_partition(p_c * consumer_task + local)?; + assert_eq!(route.consumer_task, consumer_task); + assert_eq!(route.consumer_partition, local); + } + } + sliced += 1; + Ok(TreeNodeRecursion::Continue) + })?; + assert!(sliced > 0, "the plan grew no sliced-layout boundary"); + assert!(grouped > 0, "the plan grew no per-task-group boundary"); + std::fs::remove_file(&path).ok(); + Ok(()) + } +} diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 703843cc8..4379910a5 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -182,6 +182,16 @@ impl NetworkBoundary for NetworkCoalesceExec { fn producer_head(&self, _consumer_task_count: usize) -> ProducerHead { ProducerHead::None } + + fn route_partition( + &self, + output_partition: usize, + ) -> Result { + internal_err!( + "NetworkCoalesceExec routes by producer task group, not by output partition; \ + partition {output_partition} has no slice-layout route" + ) + } } impl DisplayAs for NetworkCoalesceExec { diff --git a/src/lib.rs b/src/lib.rs index 032e56bd4..65fda1a94 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ mod codec; mod common; mod config_extension_ext; mod coordinator; +mod dispatch_plan_source; mod distributed_ext; mod distributed_planner; mod execution_plans; @@ -11,6 +12,9 @@ mod explain_analyze; mod metrics; mod passthrough_headers; mod protocol; +// Not feature-gated: the shared-memory mesh is the no-gRPC transport, so it has to build in both +// the `grpc`-on and `grpc`-off configs. +pub mod shm; mod stage; mod work_unit_feed; mod worker; @@ -23,7 +27,7 @@ pub use arrow_ipc::CompressionType; pub use coordinator::{DistributedExec, MetricsStore}; pub use distributed_ext::{DistributedExt, DistributedGetterExt}; pub use distributed_planner::{ - DistributedConfig, NetworkBoundary, NetworkBoundaryExt, SessionStateBuilderExt, + DistributedConfig, NetworkBoundary, NetworkBoundaryExt, PartitionRoute, SessionStateBuilderExt, }; pub use events::{ DesiredTaskCountEvent, DesiredTaskCountEventResponse, DesiredTaskCountHandler, RouteTasksEvent, @@ -56,6 +60,11 @@ pub use protocol::grpc; pub use protocol::generated::worker as proto; pub use codec::DistributedCodec; +pub use dispatch_plan_source::{DispatchPlanSource, get_distributed_dispatch_plan_source}; +// The producer-side sink traits live in `shm` because only a push-based transport produces through +// them; re-exported at the crate root so `crate::PartitionSink` resolves the way the shm core spells +// it. +pub use shm::{PartitionSink, WorkerSink}; pub use worker_resolver::{WorkerResolver, get_distributed_worker_resolver}; pub use protocol::{ diff --git a/src/shm/dsm.rs b/src/shm/dsm.rs new file mode 100644 index 000000000..3720da0eb --- /dev/null +++ b/src/shm/dsm.rs @@ -0,0 +1,544 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mesh-multiplexed DSM layout: one MPSC inbox per receiver process. +//! +//! Each MPP query allocates a single DSM region: +//! +//! ```text +//! +--- MppDsmHeader (repr C, MAXALIGN-padded) -------------+ +//! | magic, version, n_procs, queue_bytes | +//! | plan_offset, plan_len, queues_offset, region_total | +//! +-------------------------------------------------------+ +//! | plan bytes (bincode-serialized worker fragment) | +//! +-------------------------------------------------------+ +//! | padding to MAXALIGN | +//! +-------------------------------------------------------+ +//! | DsmMpscRing inbox array: n_procs inboxes | +//! | inbox(receiver) = queues_offset | +//! | + receiver * queue_bytes | +//! +-------------------------------------------------------+ +//! ``` +//! +//! - `n_procs` = 1 leader + N parallel workers. Leader is `proc_idx = 0`; workers are +//! `proc_idx = ParallelWorkerNumber + 1`. +//! - Each process attaches as receiver to its own inbox (one MPSC ring) and as sender +//! to each of N-1 peer inboxes. Senders stamp `MppFrameHeader::sender_proc` on every +//! frame so the receiver demuxes by source. +//! - Self-loop frames (proc → itself) ride an in-proc channel installed by +//! `worker_setup`, not a DSM slot: each `DsmMpscReceiver` is owned by a single +//! receiver process, so there is no DSM slot for the self-loop pair. + +use std::ffi::c_void; +use std::mem::size_of; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use super::AliveFlag; +use super::mesh::{align_up_maxalign_checked, aligned_queue_bytes}; +use super::mpsc_ring::{self, DsmMpscReceiver, DsmMpscRingHeader, DsmMpscSender, Wakeup}; + +/// Number of slots in each per-receiver MPSC ring. With operator-visible +/// The embedder's queue-size knob divided across `RING_SLOTS` slots, each slot holds +/// up to `(queue_bytes / RING_SLOTS) - SLOT_HEADER` bytes of payload. That's enough +/// for typical Arrow batches at the bench scales we measured. Fixed at compile time +/// for now; a future GUC could expose it if bench data points at a different sweet +/// spot. +const RING_SLOTS: u32 = 8; + +/// Floor `ring_dims_for` applies to the per-slot payload capacity. The layout minimum check +/// uses the same floor so the constructed ring always fits its inbox region. +const MIN_SLOT_CAPACITY: u32 = 64; + +/// Cache-line alignment for the per-inbox DsmMpscRing header. The ring itself is plain +/// `#[repr(C)]` (the base address is only MAXALIGN-guaranteed), but spacing the per-inbox +/// offsets to 64 keeps hot headers on separate cache lines when the base cooperates; +/// both `queues_offset` and per-inbox `queue_bytes` are aligned up to this so the +/// computed `inbox_offset(r) = queues_offset + r * queue_bytes` lands at a 64-aligned +/// address for every `r`. +const RING_ALIGN: usize = 64; + +#[inline] +fn align_up_ring(n: usize) -> Option { + let mask = RING_ALIGN - 1; + n.checked_add(mask).map(|x| x & !mask) +} + +const MPP_DSM_MAGIC: u32 = 0x4D50_5052; // "MPPR" (RPC variant) +/// Bump on any wire-incompatible change to the DSM header layout or to the inbox-offset +/// math, so attaching workers reject mismatched leaders loudly rather than reading +/// garbage. Validated in [`MppDsmHeader::validate`]. +const MPP_DSM_HEADER_VERSION: u32 = 4; + +/// Absolute cap on DSM region size. 16 GiB is two orders of magnitude beyond +/// any realistic workload; the cap fails early on a pathologically oversized +/// request rather than asking PG for ~`usize::MAX` bytes. +const MPP_DSM_MAX_BYTES: usize = 16 * 1024 * 1024 * 1024; + +/// C-repr header at offset 0 of the DSM region. +/// +/// Layout: three `u32`s + padding, then five `u64`s. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct MppDsmHeader { + pub(super) magic: u32, + pub(super) header_version: u32, + /// Total proc count. Leader is `proc_idx = 0`; workers are + /// `proc_idx = ParallelWorkerNumber + 1`. The DSM region holds `n_procs` + /// per-receiver MPSC inboxes laid out contiguously after the plan bytes. + pub n_procs: u32, + pub(super) _pad: u32, + pub(super) queue_bytes: u64, + pub(super) plan_offset: u64, + pub(super) plan_len: u64, + pub(super) queues_offset: u64, + pub region_total: u64, +} + +impl MppDsmHeader { + fn from_layout(layout: &DsmLayout) -> Self { + Self { + magic: MPP_DSM_MAGIC, + header_version: MPP_DSM_HEADER_VERSION, + n_procs: layout.n_procs, + _pad: 0, + queue_bytes: layout.queue_bytes as u64, + plan_offset: layout.plan_offset as u64, + plan_len: layout.plan_len as u64, + queues_offset: layout.queues_offset as u64, + region_total: layout.region_total as u64, + } + } + + pub(super) fn validate(&self, region_total: u64) -> Result<(), &'static str> { + if self.magic != MPP_DSM_MAGIC { + return Err("mpp: DSM header magic mismatch"); + } + if self.header_version != MPP_DSM_HEADER_VERSION { + return Err("mpp: DSM header version mismatch"); + } + if self.n_procs == 0 { + return Err("mpp: header n_procs must be > 0"); + } + if self.region_total != region_total { + return Err("mpp: DSM region_total in header disagrees with attached size"); + } + match self.plan_offset.checked_add(self.plan_len) { + None => return Err("mpp: plan_offset + plan_len overflow"), + Some(end) if end > self.queues_offset => { + return Err("mpp: plan would overlap queues area"); + } + _ => {} + } + if self.queues_offset > region_total { + return Err("mpp: queues_offset past end of region"); + } + Ok(()) + } + + /// Byte offset (relative to DSM base) of `receiver_proc`'s MPSC inbox. + /// + /// The owner attaches as receiver (`DsmMpscReceiver`); every other process attaches + /// as sender (`DsmMpscSender`) to the same inbox and stamps its identity into the + /// frame header (`MppFrameHeader::sender_proc`). + pub(super) fn inbox_offset(&self, receiver_proc: u32) -> u64 { + debug_assert!(receiver_proc < self.n_procs); + self.queues_offset + (receiver_proc as u64) * self.queue_bytes + } +} + +/// Pure-math layout for [`compute_dsm_layout`]. +#[derive(Debug, Clone, Copy)] +pub(super) struct DsmLayout { + pub n_procs: u32, + pub queue_bytes: usize, + pub plan_offset: usize, + pub plan_len: usize, + pub queues_offset: usize, + pub region_total: usize, +} + +/// Compute the DSM region size and field offsets for one MPP query. +/// +/// `n_procs` is the total proc count (1 leader + N workers); the region holds one +/// `DsmMpscRing` inbox per process. Every other process attaches as sender to that +/// inbox and the receiver demultiplexes by `MppFrameHeader::sender_proc`. +/// +/// `queue_bytes` is the per-inbox total (ring header + `RING_SLOTS` slots). +/// The embedder's operator-facing queue-size knob controls this value. +pub(super) fn compute_dsm_layout( + n_procs: u32, + queue_bytes: usize, + plan_len: usize, +) -> Result { + if n_procs < 2 { + return Err("mpp: n_procs must be >= 2 (leader + at least one worker)"); + } + // MAXALIGN-round-down first (operator-friendly), then round up to the ring's + // 64-byte alignment requirement. Doing it in this order means each per-inbox + // region is both MAXALIGN-aligned (PG DSM convention) AND cache-line aligned, + // which keeps the rings' hot header fields on separate cache lines. + let queue_bytes = aligned_queue_bytes(queue_bytes); + if queue_bytes == 0 { + return Err("mpp: queue_bytes too small after alignment"); + } + let queue_bytes = align_up_ring(queue_bytes).ok_or("mpp: queue_bytes alignment overflow")?; + // Each inbox must have room for the ring `ring_dims_for` will actually build: + // `RING_SLOTS` slots with the 64-byte capacity floor. Checking against a smaller + // ring here would let `create_at` write past the inbox region for tiny + // `queue_bytes`, overlapping the next inbox. + if queue_bytes < DsmMpscRingHeader::region_bytes(RING_SLOTS, MIN_SLOT_CAPACITY) { + return Err("mpp: queue_bytes too small for ring header + min slot capacity"); + } + let header_end = align_up_maxalign_checked(size_of::()) + .ok_or("mpp: header alignment overflow")?; + let plan_offset = header_end; + let plan_end = plan_offset + .checked_add(plan_len) + .ok_or("mpp: plan offset+len overflow")?; + // Round queues_offset up to RING_ALIGN so the first inbox starts at a 64-aligned + // address; subsequent inboxes are queue_bytes apart and queue_bytes is RING_ALIGN- + // aligned, so they all land on cache-line boundaries. + let queues_offset = align_up_ring(plan_end).ok_or("mpp: queues alignment overflow")?; + let queues_bytes = (n_procs as usize) + .checked_mul(queue_bytes) + .ok_or("mpp: queues bytes overflow")?; + let region_total = queues_offset + .checked_add(queues_bytes) + .ok_or("mpp: region total overflow")?; + if region_total > MPP_DSM_MAX_BYTES { + return Err("mpp: DSM region exceeds MPP_DSM_MAX_BYTES"); + } + Ok(DsmLayout { + n_procs, + queue_bytes, + plan_offset, + plan_len, + queues_offset, + region_total, + }) +} + +/// Derive `(ring_slots, slot_capacity)` for a per-inbox region of `queue_bytes`. +/// Total ring region size = `DsmMpscRingHeader::region_bytes(ring_slots, slot_capacity)` +/// which fits within `queue_bytes` by construction. +fn ring_dims_for(queue_bytes: usize) -> (u32, u32) { + let header = std::mem::size_of::(); + // queue_bytes >= ring_dims minimum is enforced in compute_dsm_layout; we recompute + // here without re-validating. + let slot_total_bytes = queue_bytes.saturating_sub(header); + let slot_capacity = (slot_total_bytes / RING_SLOTS as usize).max(MIN_SLOT_CAPACITY as usize); + // Cap slot_capacity at u32::MAX (DsmMpscRing's field type) to avoid casting wrap. + let slot_capacity = slot_capacity.min(u32::MAX as usize) as u32; + (RING_SLOTS, slot_capacity) +} + +/// Per-proc return from `attach_proc`: N-1 outbound senders (one per peer inbox) plus a +/// single inbound receiver (this proc's own inbox). +/// +/// The own-inbox is the multiplexed entry point: every peer attaches to it as a sender +/// (each `DsmMpscSender` increments the ring's `sender_count`) and stamps its identity +/// into `MppFrameHeader::sender_proc` on every frame. The receiver side pulls frames +/// off that single ring and routes them to per-`(sender_proc, stage_id, partition)` +/// channel buffers via [`DrainHandle`]. +pub(super) struct ProcAttach { + /// `outbound_senders[i]` writes to peer `peer_proc_for_index(this_proc, i)`'s inbox. + /// `peer_proc(i) = i if i < this_proc else i + 1` skips the self-loop entry. + pub(super) outbound_senders: Vec, + /// This process's own MPSC inbox receiver. Drained inline by `DrainHandle`. + pub(super) inbound_receiver: DsmMpscReceiver, + pub(super) alive: AliveFlag, +} + +/// Read `region_total` out of the header at the start of an initialized region. +/// +/// # Safety +/// `base` must point at the start of a region a leader wrote via [`leader_init`]. +pub(super) unsafe fn read_region_total(base: *const c_void) -> u64 { + unsafe { std::ptr::read(base as *const MppDsmHeader).region_total } +} + +/// Translate a peer index (`0..n_procs - 1`) into a process index +/// (`0..n_procs`) by skipping the self-loop slot. +#[inline] +pub(super) fn peer_proc_for_index(this_proc: u32, peer_idx: u32) -> u32 { + if peer_idx < this_proc { + peer_idx + } else { + peer_idx + 1 + } +} + +/// Initialize the DSM region as the leader (`proc_idx = 0`). Writes the MppDsmHeader, +/// copies the plan bytes, initializes the N MPSC inboxes via `DsmMpscRing::create_at`, +/// and attaches the leader as receiver to its own inbox, plus (when `attach_senders`) as +/// sender to each peer for the control plane (work-unit frames). +/// +/// `attach_senders` is a commitment: a ring latches `detached` when its sender count falls to +/// zero, so a leader that attaches and then drops its senders before a worker attached poisons +/// that worker's inbox. Attach only when the senders outlive the query. +/// +/// # Safety +/// - `coordinate` must point to the start of a DSM region of size `>= layout.region_total`. +/// - The region must be uninitialized (the leader is the first writer). +pub(super) unsafe fn leader_init( + coordinate: *mut c_void, + layout: &DsmLayout, + plan_bytes: &[u8], + wakeup: Arc, + attach_senders: bool, +) -> Result { + if coordinate.is_null() { + return Err("mpp: leader_init given null coordinate".into()); + } + if plan_bytes.len() != layout.plan_len { + return Err(format!( + "mpp: plan_bytes.len()={} != layout.plan_len={}", + plan_bytes.len(), + layout.plan_len + )); + } + + let base = coordinate as *mut u8; + + // Header. + unsafe { + std::ptr::write( + base.cast::(), + MppDsmHeader::from_layout(layout), + ); + } + // Plan bytes. + unsafe { + std::ptr::copy_nonoverlapping( + plan_bytes.as_ptr(), + base.add(layout.plan_offset), + plan_bytes.len(), + ); + } + + // Initialize the N MPSC inboxes. Workers can't do this (the region is uninitialized + // at their attach time), so the leader runs DsmMpscRing::create_at for every receiver. + let header = MppDsmHeader::from_layout(layout); + let n_procs = layout.n_procs; + let (ring_slots, slot_capacity) = ring_dims_for(layout.queue_bytes); + for r in 0..n_procs { + let off = header.inbox_offset(r) as usize; + let inbox_addr = unsafe { base.add(off) }; + unsafe { mpsc_ring::create_at(inbox_addr, ring_slots, slot_capacity) }; + } + + let attach = unsafe { attach_proc(base, &header, 0, attach_senders, wakeup) }; + Ok(attach) +} + +/// Attach to the leader-initialized DSM region as `proc_idx` (`0 = leader`, `1..N` = +/// parallel workers). Attach as receiver to this proc's own MPSC inbox, and (if +/// `attach_senders` is true) as sender to every peer's inbox. +/// +/// `attach_senders = false` is for the leader (consumer-only). If the leader attached +/// as sender it would bump every peer inbox's `sender_count` and decrement it on `Drop`; +/// if it dropped before any worker bumped, the 1 → 0 transition would flip `detached` +/// on every peer inbox and every later worker send would fail `SendError::Detached`. +/// Skipping keeps `sender_count` honest (only producers ever increment). +/// +/// # Safety +/// - `base` must point to a DSM region whose header has been validated. +/// - `header.inbox_offset(r)` must point at a ring already initialized by +/// `DsmMpscRing::create_at` (the leader does this in `leader_init`). +unsafe fn attach_proc( + base: *mut u8, + header: &MppDsmHeader, + this_proc: u32, + attach_senders: bool, + wakeup: Arc, +) -> ProcAttach { + let n_procs = header.n_procs; + let peer_count = (n_procs - 1) as usize; + let mut outbound_senders = Vec::with_capacity(if attach_senders { peer_count } else { 0 }); + + let (ring_slots, slot_capacity) = ring_dims_for(header.queue_bytes as usize); + + // One liveness flag shared by every handle this attach mints. + let alive = Arc::new(AtomicBool::new(true)); + + if attach_senders { + for peer_idx in 0..(n_procs - 1) { + let r = peer_proc_for_index(this_proc, peer_idx); + let off = header.inbox_offset(r) as usize; + let inbox_addr = unsafe { base.add(off) }; + let nn = unsafe { mpsc_ring::attach_at(inbox_addr, ring_slots, slot_capacity) } + .expect("DsmMpscRing attach_at: leader-initialized region must validate"); + outbound_senders + .push(unsafe { DsmMpscSender::new(nn, Arc::clone(&wakeup), Arc::clone(&alive)) }); + } + } + + // Inbound: this proc's own inbox. Single receiver per ring (MPSC contract). + let own_off = header.inbox_offset(this_proc) as usize; + let own_inbox_addr = unsafe { base.add(own_off) }; + let own_nn = unsafe { mpsc_ring::attach_at(own_inbox_addr, ring_slots, slot_capacity) } + .expect("DsmMpscRing attach_at: own inbox must validate"); + let inbound_receiver = unsafe { DsmMpscReceiver::new(own_nn, Arc::clone(&alive)) }; + + ProcAttach { + outbound_senders, + inbound_receiver, + alive, + } +} + +/// Attach to the leader-initialized DSM region as `proc_idx` (1-based for +/// workers: PG's `ParallelWorkerNumber + 1`). +/// +/// # Safety +/// - `coordinate` must be the DSM region pointer the leader initialized. +/// - `region_total` must match the DSM's attached size. +pub(super) unsafe fn worker_attach( + coordinate: *mut c_void, + region_total: u64, + proc_idx: u32, + wakeup: Arc, +) -> Result<(MppDsmHeader, Vec, ProcAttach), String> { + if coordinate.is_null() { + return Err("mpp: worker_attach given null coordinate".into()); + } + let base = coordinate as *mut u8; + let header = unsafe { std::ptr::read(base.cast::()) }; + header + .validate(region_total) + .map_err(|e| format!("mpp: worker DSM validate: {e}"))?; + if proc_idx == 0 { + return Err( + "mpp: worker_attach must be called with proc_idx >= 1 (proc 0 is leader)".into(), + ); + } + if proc_idx >= header.n_procs { + return Err(format!( + "mpp: proc_idx {proc_idx} not in 1..{}", + header.n_procs + )); + } + + // Copy plan bytes out of DSM so the caller has an owned buffer. + let plan_bytes = unsafe { + std::slice::from_raw_parts( + base.add(header.plan_offset as usize), + header.plan_len as usize, + ) + .to_vec() + }; + + let attach = unsafe { + attach_proc( + base, &header, proc_idx, /* attach_senders */ true, wakeup, + ) + }; + Ok((header, plan_bytes, attach)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compute_dsm_layout_works() { + let l = compute_dsm_layout(4, 64 * 1024, 1024).unwrap(); + assert_eq!(l.n_procs, 4); + // 4 procs => 4 inboxes laid out contiguously after the plan bytes. + let aligned = aligned_queue_bytes(64 * 1024); + let queues_size = 4 * aligned; + assert_eq!(l.region_total, l.queues_offset + queues_size); + } + + #[test] + fn compute_dsm_layout_rejects_zero_procs() { + assert!(compute_dsm_layout(0, 64 * 1024, 0).is_err()); + } + + #[test] + fn compute_dsm_layout_rejects_oversize() { + assert!(compute_dsm_layout(u32::MAX, 64 * 1024, 0).is_err()); + } + + #[test] + fn compute_dsm_layout_scales_linearly_in_n_procs() { + // Total queue area must grow as O(N) in proc count. Pinning the math here so a + // regression that grows queues per-pair fails this test at compile time rather + // than at the N=24 wall-time cliff in production. + let queue_bytes = 64 * 1024; + let aligned = aligned_queue_bytes(queue_bytes); + for n in [2u32, 4, 8, 16, 24] { + let l = compute_dsm_layout(n, queue_bytes, 0).unwrap(); + let expected = (n as usize) * aligned; + assert_eq!( + l.region_total - l.queues_offset, + expected, + "n={n}: expected {expected} inbox bytes ({n} inboxes × {aligned})" + ); + } + } + + #[test] + fn header_inbox_offset_is_per_receiver() { + // 4 procs => 4 inboxes, contiguous, sized by queue_bytes each. + let l = compute_dsm_layout(4, 64 * 1024, 0).unwrap(); + let h = MppDsmHeader::from_layout(&l); + let aligned = h.queue_bytes; + assert_eq!(h.inbox_offset(0), h.queues_offset); + assert_eq!(h.inbox_offset(1), h.queues_offset + aligned); + assert_eq!(h.inbox_offset(2), h.queues_offset + 2 * aligned); + assert_eq!(h.inbox_offset(3), h.queues_offset + 3 * aligned); + } + + #[test] + fn header_validate_accepts_self() { + let l = compute_dsm_layout(2, 64 * 1024, 0).unwrap(); + let h = MppDsmHeader::from_layout(&l); + assert!(h.validate(l.region_total as u64).is_ok()); + } + + #[test] + fn header_validate_rejects_wrong_version() { + let l = compute_dsm_layout(2, 64 * 1024, 0).unwrap(); + let mut h = MppDsmHeader::from_layout(&l); + h.header_version = MPP_DSM_HEADER_VERSION.wrapping_sub(1); + let err = h + .validate(l.region_total as u64) + .expect_err("wrong version must fail"); + assert!(err.contains("DSM header version mismatch"), "got: {err}"); + } + + #[test] + fn header_validate_rejects_size_mismatch() { + let l = compute_dsm_layout(2, 64 * 1024, 0).unwrap(); + let h = MppDsmHeader::from_layout(&l); + assert!(h.validate(l.region_total as u64 + 1).is_err()); + } + + /// Pins the coupling between `compute_dsm_layout`'s minimum check and `ring_dims_for`'s + /// slot-capacity floor: a queue sized below the floored ring must be rejected, or + /// `create_at` would write past its inbox region. + #[test] + fn layout_rejects_queue_smaller_than_floored_ring() { + let floored = DsmMpscRingHeader::region_bytes(RING_SLOTS, MIN_SLOT_CAPACITY); + let too_small = DsmMpscRingHeader::region_bytes(RING_SLOTS, 1); + assert!(too_small < floored); + assert!(compute_dsm_layout(2, too_small, 0).is_err()); + assert!(compute_dsm_layout(2, floored.next_multiple_of(64), 0).is_ok()); + } +} diff --git a/src/shm/in_process.rs b/src/shm/in_process.rs new file mode 100644 index 000000000..4de15adbf --- /dev/null +++ b/src/shm/in_process.rs @@ -0,0 +1,1109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! In-process instantiation of the shared-memory transport, plus an end-to-end test. +//! +//! A real distributed query runs through [`ShmChannelResolver`] with no Postgres and no Flight. +//! This is the rebase-safety payoff: a single process plays every role that production splits +//! across the leader (build and slice the plan), Postgres (allocate the DSM region, launch the +//! workers), and each worker (run a producer fragment and push it through the mesh). If an upstream +//! change breaks the channel-protocol contract, the boundary routing API, or the ready-to-run +//! fragment contract, the test below fails here, before any downstream embedder rebuilds. +//! +//! Workers are passive executors: they run the ready-to-run per-task plans the coordinator +//! dispatches (task-specialized, nested stages `Remote`), whose boundary leaves read the mesh +//! through the session's [`ShmChannelResolver`]. Nothing on the worker re-plans, converts, or +//! dispatches. A [`DispatchPlanSource`] on the leader session captures each dispatched plan, +//! standing in for a production source that serializes it with the embedder's codec. +//! +//! What's faithful vs. simplified relative to the Postgres path: +//! - Faithful: the DSM ring mesh, the framing, the cooperative drain, `ShmWorkerChannel::execute_task`, +//! the per-fragment routing (`collect_dispatched_stages`), `run_worker_fragment`, the leader +//! consuming via `DistributedExec::execute`, and the dispatch handoff through +//! `DispatchPlanSource`. +//! - Simplified: the dispatched subplans are captured as `Arc`s rather than serialized through +//! the `SetPlan` frames the coordinator routes over the mesh (all roles live in one address +//! space), the wakeup is a no-op (the cooperative consumer yields rather than parking), and +//! there's no cancellation source. + +use std::alloc::Layout; +use std::ffi::c_void; +use std::sync::{Arc, Mutex}; + +use datafusion::arrow::array::{Int32Array, RecordBatch, StringArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::catalog::memory::DataSourceExec; +use datafusion::common::runtime::JoinSet; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::common::{DataFusionError, HashMap, Result}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::execution::{SessionStateBuilder, TaskContext}; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion::prelude::{SessionConfig, SessionContext}; + +use crate::{ + DispatchPlanSource, DistributedConfig, DistributedExec, DistributedExt, DistributedLeafExec, + DistributedTaskContext, NetworkBoundaryExt, NetworkBroadcastExec, NetworkCoalesceExec, + NetworkShuffleExec, PartitionSink, SessionStateBuilderExt, Stage, TaskKey, WorkerSink, + decode_task_metrics, +}; + +use super::mpsc_ring::Wakeup; +use super::runtime::{InProcessWorkerResolver, MppMesh, ShmChannelResolver, proc_for_task}; +use super::setup::{ + collect_task_metrics, dsm_region_bytes, leader_setup, run_worker_fragment, worker_setup, +}; +use super::transport::{ + CooperativeDrainSet, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, +}; + +/// Per-inbox DSM ring size for the in-process mesh. Generous: the test ships a handful of tiny +/// batches, so backpressure never kicks in. Production sizes this from `paradedb.mpp_queue_size`. +const IN_PROCESS_QUEUE_BYTES: usize = 1 << 20; + +/// No-op wakeup. The cooperative consumer in `runtime::pull_partition_stream` yields rather +/// than parking, so a publish never needs to wake a blocked thread. The real cross-thread wakeup +/// extension point is covered by `mpsc_ring`'s `injected_wakeup_unparks_blocked_consumer`. +struct NoopWakeup; +impl Wakeup for NoopWakeup { + fn wake(&self, _token: u64) {} +} + +/// Owns the single heap buffer that stands in for the PG DSM segment. Every proc (leader + workers) +/// reads and writes the same region through raw pointers; the lock-free rings make concurrent +/// access sound. Kept alive in the harness until after all producer tasks join. +struct HeapRegion { + ptr: *mut u8, + layout: Layout, +} + +impl HeapRegion { + fn new(bytes: usize) -> Self { + // 64-byte alignment so each per-inbox ring header lands on its own cache line; the + // dsm layout aligns the offsets within the region, but only if the base is aligned too. + let layout = Layout::from_size_align(bytes, 64).expect("dsm region layout"); + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null(), "dsm region alloc failed"); + Self { ptr, layout } + } + + fn base(&self) -> *mut c_void { + self.ptr as *mut c_void + } +} + +impl Drop for HeapRegion { + fn drop(&mut self) { + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } +} + +/// Send wrapper so the leader-init / worker-attach raw base pointer can be handed to the per-proc +/// setup. All setup runs on the harness thread, so the pointer never crosses into a spawned task; +/// only the resulting `Send` meshes and senders do. +#[derive(Clone, Copy)] +struct SharedBase(*mut c_void); +unsafe impl Send for SharedBase {} + +/// Opaque, non-sentinel receiver token. `NoopWakeup` ignores the value; this just exercises the +/// `set_receiver` path with something the producer won't skip as "no consumer registered". +fn receiver_token(proc_idx: u32) -> u64 { + proc_idx as u64 + 1 +} + +// --------------------------------------------------------------------------------------------- +// Leader-side producer-stage discovery (in-process port of pg_search's `collect_dispatched_stages`). +// The crate's gRPC path keys dispatch on resolver URLs and never decides this; the shm_mq peers are +// push-driven without URLs, so the embedder classifies each boundary's routing here. +// --------------------------------------------------------------------------------------------- + +/// Routing rule for a producer fragment's output partitions. +#[derive(Clone, Debug)] +enum FragmentRouting { + /// Every output partition goes to one destination proc (a `NetworkCoalesceExec`, or the + /// top-level gather to the leader). + Coalesce { dest_proc: u32 }, + /// Hash-partitioned mesh (`NetworkShuffleExec` / `NetworkBroadcastExec`): output partition `q` + /// goes to the consumer task the crate's `route_partition(q)` selects. + Hashed { + consumer_task: Vec, + broadcast: bool, + }, +} + +/// One producer stage's routing metadata, captured from a network boundary. The stage's plan is +/// not captured here: workers run the specialized plans the coordinator dispatches, delivered +/// through the leader session's [`CapturingPlanSource`]. +struct StageEntry { + stage_num: u32, + task_count: usize, + routing: FragmentRouting, +} + +/// The dispatched plans, keyed by `(stage_id, task_number)`. Filled by [`CapturingPlanSource`] +/// while the leader's `execute` prepares the query; read by the worker tasks. One query per +/// mesh, so the key drops the query id. +type CapturedPlans = Arc>>>; + +/// Test-harness [`DispatchPlanSource`]: records each `(stage, task)` specialized plan the +/// coordinator hands over, standing in for a production source that serializes it with the +/// embedder's codec. The returned empty bytes keep the coordinator from encoding plans nobody +/// decodes; the workers run the captured `Arc`s directly. +struct CapturingPlanSource(CapturedPlans); + +impl DispatchPlanSource for CapturingPlanSource { + fn dispatch_plan_proto( + &self, + task: &crate::TaskKey, + specialized: &Arc, + ) -> Option>> { + self.0 + .lock() + .unwrap() + .insert((task.stage_id, task.task_number), Arc::clone(specialized)); + Some(Ok(Vec::new())) + } +} + +/// Walk the distributed physical plan and collect every producer stage, once per boundary. +fn collect_dispatched_stages(root: &Arc, n_workers: u32) -> Vec { + let mut out = Vec::new(); + collect_stages(root, n_workers, /* nested = */ false, &mut out); + out +} + +fn collect_stages( + plan: &Arc, + n_workers: u32, + nested: bool, + out: &mut Vec, +) { + if let Some(nb) = plan.as_ref().as_network_boundary() { + let stage = nb.input_stage(); + let stage_id = stage.num() as u32; + let route_consumer_tasks = || { + // The crate owns the consumer-slice layout, so the push side reads it from + // `route_partition` instead of re-deriving it and drifting when the layout changes. + let n_out = stage + .local_plan() + .map_or(0, |p| p.properties().partitioning.partition_count()); + (0..n_out) + .map(|q| { + nb.route_partition(q) + .expect("route_partition") + .consumer_task as u32 + }) + .collect::>() + }; + let plan_any = plan.as_ref(); + let routing = if plan_any.is::() { + if nested { + FragmentRouting::Coalesce { + dest_proc: proc_for_task(n_workers, 0), + } + } else { + FragmentRouting::Coalesce { dest_proc: 0 } + } + } else if plan_any.is::() { + assert!(nested, "top-level NetworkShuffleExec is unsupported"); + FragmentRouting::Hashed { + consumer_task: route_consumer_tasks(), + broadcast: false, + } + } else if plan_any.is::() { + assert!(nested, "top-level NetworkBroadcastExec is unsupported"); + FragmentRouting::Hashed { + consumer_task: route_consumer_tasks(), + broadcast: true, + } + } else { + panic!("unrecognized network boundary {}", plan.name()); + }; + + let task_count = stage.task_count(); + if let Some(stage_plan) = stage.local_plan() { + out.push(StageEntry { + stage_num: stage_id, + task_count, + routing, + }); + // The boundary's children() returns [stage.plan], so descending here would double-count + // every nested stage. Recurse through the stage plan directly with nested = true. + collect_stages(stage_plan, n_workers, true, out); + } + return; + } + for child in plan.children() { + collect_stages(child, n_workers, nested, out); + } +} + +/// One producer fragment assigned to a worker proc: a single task of a producer stage. The plan +/// arrives separately, through the [`CapturedPlans`] map the leader's dispatch fills. +struct FragmentAssignment { + stage_id: u32, + task_idx: usize, + task_count: usize, + routing: FragmentRouting, +} + +/// Rebuild a plan subtree so each fragment executes its own node instances. In production every +/// proc decodes its own copy of the plan; a captured `Arc` is shared with the coordinator and +/// with sibling tasks of the same stage, and sharing execute-once state breaks (a +/// `RepartitionExec` panics when a second fragment executes a partition the first already +/// consumed; a boundary's connection pool hands each partition stream out once). A `Remote` +/// boundary is childless, so it gets a fresh node explicitly; other leaves are shared (they +/// carry no execute-once state here). +fn reinstantiate(plan: &Arc) -> Arc { + if let Some(nb) = plan.as_ref().as_network_boundary() + && matches!(nb.input_stage(), Stage::Remote(_)) + { + return nb + .with_input_stage(nb.input_stage().clone()) + .expect("with_input_stage with the same stage"); + } + let children: Vec<_> = plan.children().into_iter().map(reinstantiate).collect(); + if children.is_empty() { + Arc::clone(plan) + } else { + Arc::clone(plan) + .with_new_children(children) + .expect("with_new_children with the same arity") + } +} + +/// Wait for the leader's dispatch to capture this fragment's plan. The capture happens inside +/// the leader's `execute` (during plan preparation), which interleaves with the worker tasks on +/// the cooperative runtime. Bounded so a dispatch that never happens fails the test instead of +/// hanging it. +async fn captured_plan( + captured: &CapturedPlans, + stage_id: u32, + task_idx: usize, +) -> Arc { + for _ in 0..100_000 { + if let Some(plan) = captured + .lock() + .unwrap() + .get(&(stage_id as usize, task_idx)) + .cloned() + { + return plan; + } + tokio::task::yield_now().await; + } + panic!("no dispatched plan captured for stage {stage_id} task {task_idx}"); +} + +/// Expand the dispatched stages into the fragments `this_proc` owns under `proc_for_task`. +fn fragments_for_proc( + entries: &[StageEntry], + this_proc: u32, + n_workers: u32, +) -> Vec { + let mut out = Vec::new(); + for entry in entries { + for task_idx in 0..entry.task_count { + if proc_for_task(n_workers, task_idx as u32) != this_proc { + continue; + } + // Broadcast caps its build subtree at task 0; the other tasks would re-emit the same + // canonical replica and the consumer's select_all would over-count. + if matches!( + entry.routing, + FragmentRouting::Hashed { + broadcast: true, + .. + } + ) && task_idx != 0 + { + continue; + } + out.push(FragmentAssignment { + stage_id: entry.stage_num, + task_idx, + task_count: entry.task_count, + routing: entry.routing.clone(), + }); + } + } + out +} + +/// Build a fragment's `TaskContext`, carrying the right `DistributedTaskContext` so nested boundary +/// nodes know their `(task_index, task_count)` and the worker session's channel resolver rides +/// along for their mesh reads. +fn fragment_task_ctx( + session: &SessionContext, + task_index: usize, + task_count: usize, +) -> Arc { + let cfg = session + .state() + .config() + .clone() + .with_extension(Arc::new(DistributedTaskContext { + task_index, + task_count, + })); + Arc::new(TaskContext::default().with_session_config(cfg)) +} + +/// Run all fragments owned by one worker proc, then signal completion. Mirrors the body of +/// pg_search's `run_mpp_worker`: build a [`WorkerSink`] that routes by partition, open a +/// [`PartitionSink`] per output partition, execute each dispatched fragment as-is (it arrives +/// ready-to-run, nested stages `Remote`, boundary leaves reading the mesh), and join. +async fn run_worker_proc( + fragments: Vec, + outbound: Vec>, + mesh: Arc, + session: SessionContext, + n_workers: u32, + captured: CapturedPlans, +) -> Result<()> { + let mut routing = HashMap::new(); + for fragment in &fragments { + routing.insert(fragment.stage_id, fragment.routing.clone()); + } + // One sink serves every stage this proc produces; it owns the base outbound senders and routes + // each (stage, partition) to the destination proc's send end. + let worker_sink = ShmMqWorkerSink { + outbound, + mesh: Arc::clone(&mesh), + n_workers, + routing, + }; + + let mut prepared = Vec::with_capacity(fragments.len()); + for fragment in &fragments { + let task_ctx = fragment_task_ctx(&session, fragment.task_idx, fragment.task_count); + // Production decodes a fresh plan per fragment; the captured Arc is shared, so copy it. + let plan = + reinstantiate(&captured_plan(&captured, fragment.stage_id, fragment.task_idx).await); + let n_out = plan.output_partitioning().partition_count(); + let mut sinks: Vec> = Vec::with_capacity(n_out); + for q in 0..n_out { + sinks.push(worker_sink.open_partition(fragment.stage_id as usize, q)?); + } + prepared.push((fragment, plan, sinks, task_ctx)); + } + // The metrics frames go to the leader after the fragments finish; the clone keeps one sender + // on the leader's inbox alive past the drop below, which only delays that ring's detach + // observation, never a per-channel EOF. + let metrics_sender_base = worker_sink + .outbound + .first() + .and_then(|s| s.as_ref()) + .map(|s| s.clone_with_header(MppFrameHeader::task_metrics(0, 0, mesh.this_proc))); + // Drop the base senders so the only senders left are the per-partition clones the fragment + // futures own; otherwise the rings never observe the last-sender detach. + drop(worker_sink); + + let mut futures = Vec::with_capacity(prepared.len()); + let mut executed = Vec::with_capacity(prepared.len()); + for (fragment, plan, sinks, task_ctx) in prepared { + executed.push(( + fragment.stage_id, + fragment.task_idx, + fragment.task_count, + Arc::clone(&plan), + )); + futures.push(run_worker_fragment(plan, sinks, task_ctx)); + } + for r in futures::future::join_all(futures).await { + r?; + } + // Report per-fragment metrics, the same frames pg's workers ship after their last EOF; the + // leader files them into the executed plan's metrics store. + if let Some(base) = metrics_sender_base { + for (stage_id, task_idx, task_count, plan) in &executed { + let frame = collect_task_metrics(plan, *task_idx, *task_count); + let sender = base.clone_with_header(MppFrameHeader::task_metrics( + *stage_id, + *task_idx as u32, + mesh.this_proc, + )); + let _ = sender.send_task_metrics_best_effort(&frame).await; + } + } + Ok(()) +} + +/// Test-harness [`WorkerSink`]: routes each `(stage, partition)` to the destination proc's outbound +/// send end, the in-process analog of what pg_search builds on a real backend. Holds the base +/// senders plus the per-stage routing so `open_partition` reproduces the header + cooperative-drain +/// wiring the produce loop used to apply inline. +struct ShmMqWorkerSink { + outbound: Vec>, + mesh: Arc, + n_workers: u32, + routing: HashMap, +} + +impl WorkerSink for ShmMqWorkerSink { + fn open_partition(&self, stage: usize, partition: usize) -> Result> { + let routing = self.routing.get(&(stage as u32)).ok_or_else(|| { + DataFusionError::Internal(format!("run_worker_proc: no routing for stage {stage}")) + })?; + let dest_proc = match routing { + FragmentRouting::Coalesce { dest_proc } => *dest_proc, + FragmentRouting::Hashed { consumer_task, .. } => { + proc_for_task(self.n_workers, consumer_task[partition]) + } + }; + let base = self.outbound[dest_proc as usize].as_ref().ok_or_else(|| { + DataFusionError::Internal(format!( + "run_worker_proc: no outbound sender for dest proc {dest_proc}" + )) + })?; + let sender = base + .clone_with_header(MppFrameHeader::batch( + stage as u32, + partition as u32, + self.mesh.this_proc, + )) + .with_cooperative_drain(Arc::clone(&self.mesh) as Arc); + Ok(Box::new(MppPartitionSink::new(sender))) + } +} + +/// Splits an in-memory `DataSourceExec` leaf across tasks, the in-memory analog of the crate's +/// `FileScanConfigTaskEstimator` (which only handles file scans). Each task reads a disjoint subset +/// of the source's partitions, so a gather over the tasks reproduces the serial result exactly. +#[derive(Debug)] +struct MemShardConfig; + +impl MemShardConfig { + fn mem_source(plan: &Arc) -> Option<&MemorySourceConfig> { + plan.downcast_ref::()? + .data_source() + .downcast_ref::() + } +} + +fn mem_shard_desired_task_count( + ev: crate::events::DesiredTaskCountEvent, +) -> Option> { + let n_tasks = ev.session_config.target_partitions(); + MemShardConfig::mem_source(ev.plan).map(|_| { + Ok(crate::events::DesiredTaskCountEventResponse::desired( + n_tasks, + )) + }) +} + +fn mem_shard_scale_up_leaf_node( + ev: crate::events::ScaleUpLeafNodeEvent, +) -> Option> { + let task_count = ev.task_count; + if task_count <= 1 { + return None; + } + let mem = MemShardConfig::mem_source(ev.plan)?; + let parts = mem.partitions().to_vec(); + let n_part = parts.len(); + let unprojected_schema: SchemaRef = parts + .iter() + .flatten() + .next() + .map(|b| b.schema()) + .unwrap_or_else(|| ev.plan.schema()); + let projection = mem.projection().clone(); + let variants = (0..task_count).map(|i| { + let per_task: Vec> = (0..n_part) + .map(|j| { + if j % task_count == i { + parts[j].clone() + } else { + Vec::new() + } + }) + .collect(); + MemorySourceConfig::try_new_exec(&per_task, unprojected_schema.clone(), projection.clone()) + .expect("memory variant") as Arc + }); + match DistributedLeafExec::try_new(Arc::clone(ev.plan), variants) { + Ok(plan) => Some(Ok(crate::events::ScaleUpLeafNodeEventResponse { + plan: Arc::new(plan), + })), + Err(e) => Some(Err(e)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::datasource::MemTable; + use futures::TryStreamExt; + + /// Total procs = leader (proc 0) + `N_WORKERS` producers. With round-robin `proc_for_task`, + /// worker proc `p` runs producer task `p - 1`. + const N_WORKERS: u32 = 3; + + fn table_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("val", DataType::Int32, false), + ])) + } + + /// `N_WORKERS` partitions, two rows each, so the shard estimator hands one partition per task. + fn table_partitions() -> Vec> { + let schema = table_schema(); + (0..N_WORKERS as i32) + .map(|p| { + let ids = Int32Array::from(vec![p * 2, p * 2 + 1]); + let vals = Int32Array::from(vec![p * 20, p * 20 + 10]); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vals)]) + .unwrap(); + vec![batch] + }) + .collect() + } + + fn register_table(ctx: &SessionContext) { + let table = MemTable::try_new(table_schema(), table_partitions()).unwrap(); + ctx.register_table("t", Arc::new(table)).unwrap(); + } + + fn wide_table_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("val", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ])) + } + + /// High-cardinality groups over ~1 KiB strings: each worker's partial-aggregate state runs + /// to hundreds of KiB, so its emit slices balloon past a tiny ring through arrow-ipc. + fn wide_table_partitions() -> Vec> { + let schema = wide_table_schema(); + const ROWS_PER_PART: i32 = 700; + (0..N_WORKERS as i32) + .map(|p| { + let vals = + Int32Array::from_iter_values((0..ROWS_PER_PART).map(|i| p * ROWS_PER_PART + i)); + let strings = StringArray::from_iter_values( + (0..ROWS_PER_PART).map(|i| format!("{p:02}-{i:06}-{}", "x".repeat(1024))), + ); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(vals), Arc::new(strings)]) + .unwrap(); + vec![batch] + }) + .collect() + } + + /// Swap the session's `t` for the wide-string variant. `build_session` registers the + /// standard table; the swap only matters on the leader (fragments travel as shared plan + /// Arcs, so worker sessions never re-resolve the table). + fn register_wide_table(ctx: &SessionContext) { + let _ = ctx.deregister_table("t").unwrap(); + let table = MemTable::try_new(wide_table_schema(), wide_table_partitions()).unwrap(); + ctx.register_table("t", Arc::new(table)).unwrap(); + } + + /// A worker is a consumer too (it reads shuffle inputs), so when one of its input streams drops + /// early it has to cancel that stream's producer, not just the leader. This checks the wiring + /// end-to-end: a worker proc's `cancel_stream` reaches the producing proc's inbox through the + /// control senders `worker_setup` installs, and the producer records it. The producer then ends + /// the stream cleanly, which `producer_send_ends_when_consumer_cancels_the_stream` covers. + #[test] + fn worker_consumer_cancels_its_producer() { + use crate::shm::transport::CooperativeDrainSet; + + // procs: leader 0, plus workers 1 and 2. + let boot = bootstrap_mesh(3); + let consumer = Arc::clone(&boot.workers[0].1); // proc 1 + let producer = Arc::clone(&boot.workers[1].1); // proc 2 + + assert!(!producer.stream_cancelled(7, 0)); + + // Proc 1 abandons the `(stage 7, partition 0)` stream it reads from proc 2. + consumer.cancel_stream(2, 7, 0); + + // Proc 2 drains its inbox and sees the cancel its consumer sent. + producer.try_drain_pass().unwrap(); + assert!(producer.stream_cancelled(7, 0)); + // Scoped to that one stream: a sibling partition stays live. + assert!(!producer.stream_cancelled(7, 1)); + } + + /// `dispatch_capture` is Some on the leader session only: its coordinator is the one that + /// dispatches, and the capturing source stands in for a production embedder's serializing + /// source. + fn build_session( + mesh: Arc, + dispatch_capture: Option, + ) -> SessionContext { + let config = SessionConfig::new().with_target_partitions(N_WORKERS as usize); + let mut builder = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_option_extension(DistributedConfig::default()) + .with_distributed_worker_resolver(InProcessWorkerResolver::new(N_WORKERS as usize)) + .with_distributed_channel_resolver(ShmChannelResolver::new(mesh)) + .with_distributed_planner(); + if let Some(captured) = dispatch_capture { + builder = builder + .with_distributed_dispatch_plan_source(CapturingPlanSource(captured)) + .with_distributed_metrics_collection(true) + .expect("with_distributed_metrics_collection"); + } + let cfg = builder.config().get_or_insert_default(); + crate::events::DesiredTaskCountHandlers::push_builtin( + cfg, + Arc::new(mem_shard_desired_task_count), + ); + crate::events::ScaleUpLeafNodeHandlers::push_builtin( + cfg, + Arc::new(mem_shard_scale_up_leaf_node), + ); + let state = builder.build(); + let ctx = SessionContext::new_with_state(state); + register_table(&ctx); + ctx + } + + fn new_captured_plans() -> CapturedPlans { + Arc::new(Mutex::new(HashMap::default())) + } + + /// Collect the `id` column out of a batch list, in row order. + fn ids_of(batches: &[RecordBatch]) -> Vec { + let mut out = Vec::new(); + for b in batches { + let col = b + .column(0) + .as_any() + .downcast_ref::() + .expect("id column"); + out.extend((0..col.len()).map(|i| col.value(i))); + } + out + } + + /// A real distributed query runs end-to-end through the shm_mq transport with no Postgres and + /// no Flight: producer fragments push through the heap-backed mesh while the leader gathers via + /// `DistributedExec::execute`. The gathered, ordered result must match the serial reference. + /// + /// Single-threaded runtime on purpose: it mirrors the per-process current-thread runtime each + /// PG worker runs, and it's exactly the cooperative model the transport is built for (the + /// producer send spin and the consumer pull loop interleave by yielding, not by parallelism). + #[tokio::test(flavor = "current_thread")] + async fn in_process_distributed_query_matches_serial() { + let query = "SELECT id, val FROM t ORDER BY id"; + + // Serial reference: same query, no distribution. + let serial_ctx = SessionContext::new(); + register_table(&serial_ctx); + let expected = serial_ctx + .sql(query) + .await + .unwrap() + .collect() + .await + .unwrap(); + let expected_ids = ids_of(&expected); + assert_eq!(expected_ids, vec![0, 1, 2, 3, 4, 5]); + + // One heap region stands in for the DSM segment; size it for n_procs = leader + workers. + let n_procs = N_WORKERS + 1; + let region_total = dsm_region_bytes(n_procs, IN_PROCESS_QUEUE_BYTES, 0).unwrap(); + let region = HeapRegion::new(region_total); + let base = SharedBase(region.base()); + let wakeup: Arc = Arc::new(NoopWakeup); + + // Leader first (it initializes the rings), then each worker attaches. No plan bytes travel + // through the region here; all roles share the producer subplans as Arcs. + let leader_mesh = unsafe { + leader_setup( + base.0, + n_procs, + IN_PROCESS_QUEUE_BYTES, + &[], + Arc::clone(&wakeup), + receiver_token(0), + Arc::new(NoInterrupt), + /* attach_senders */ false, + ) + } + .unwrap() + .mesh; + let mut worker_setups = Vec::new(); + for proc_idx in 1..n_procs { + let attach = unsafe { + worker_setup( + base.0, + region_total, + proc_idx, + Arc::clone(&wakeup), + receiver_token(proc_idx), + Arc::new(NoInterrupt), + ) + } + .unwrap(); + worker_setups.push((proc_idx, attach.mesh, attach.outbound_senders)); + } + + // Build the distributed plan once on the leader session; producers and consumer share it. + let captured = new_captured_plans(); + let leader_ctx = build_session(Arc::clone(&leader_mesh), Some(Arc::clone(&captured))); + let physical = leader_ctx + .sql(query) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + assert!( + physical.is::(), + "expected a DistributedExec root, got {}", + physical.name() + ); + + let entries = collect_dispatched_stages(&physical, N_WORKERS); + // Guard against a planner change that silently collapses the query to a trivial one-task + // gather: the producer stage must actually fan across every worker, or the transport's + // multi-task routing never gets exercised. + assert!( + entries.iter().any(|e| e.task_count == N_WORKERS as usize), + "expected a producer stage with task_count = {N_WORKERS}; got {:?}", + entries.iter().map(|e| e.task_count).collect::>() + ); + + // Launch the producer fragments before the leader pulls, so the mesh has data flowing while + // the consumer drains. On the current-thread runtime the spawned tasks interleave with the + // consumer by yielding, the same cooperative model each PG worker runs. + let mut workers = JoinSet::new(); + for (proc_idx, mesh, outbound) in worker_setups { + let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); + let session = build_session(Arc::clone(&mesh), None); + workers.spawn(run_worker_proc( + fragments, + outbound, + mesh, + session, + N_WORKERS, + Arc::clone(&captured), + )); + } + + // Leader consumer: execute the DistributedExec root, same as the production embedder; the + // network boundary nodes pull from the mesh through ShmWorkerChannel::execute_task. + let leader_task_ctx = leader_ctx.task_ctx(); + let stream = physical.execute(0, leader_task_ctx).unwrap(); + let got: Vec = stream.try_collect().await.unwrap(); + + while let Some(res) = workers.join_next().await { + res.expect("worker task panicked").expect("worker proc"); + } + + let got_ids = ids_of(&got); + assert_eq!(got_ids, expected_ids, "distributed gather != serial"); + + // The workers shipped one metrics frame per fragment over the mesh; file them into the + // executed plan's store the way the pg embedder does, and require full coverage. + let dist = physical + .downcast_ref::() + .expect("DistributedExec"); + let store = dist.metrics_store().expect("metrics collection enabled"); + let mut query_id = None; + let mut expected_reports = 0usize; + let _ = physical.apply(|node| { + if let Some(nb) = node.as_ref().as_network_boundary() { + let stage = nb.input_stage(); + query_id.get_or_insert_with(|| stage.query_id()); + expected_reports += stage.task_count(); + } + Ok(TreeNodeRecursion::Continue) + }); + let query_id = query_id.expect("a distributed plan has at least one boundary"); + let mut rx = leader_mesh + .take_task_metrics_receiver() + .expect("task metrics receiver"); + let mut inserted = 0usize; + for _ in 0..1_000 { + let _ = leader_mesh.try_drain_pass(); + while let Ok((stage_id, task_number, metrics)) = rx.try_recv() { + let metrics = decode_task_metrics(metrics).expect("decode task metrics"); + store.insert( + TaskKey { + query_id, + stage_id: stage_id as usize, + task_number: task_number as usize, + }, + metrics, + ); + inserted += 1; + } + if inserted >= expected_reports { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + inserted, expected_reports, + "every producer task reports metrics" + ); + + // `region` is declared before the meshes, so reverse drop order frees it after every + // receiver handle into it is gone. + } + + /// Mesh bootstrap shared by the tests: leader first (it initializes the rings), then each + /// worker attaches. + struct Bootstrap { + leader_mesh: Arc, + workers: Vec<(u32, Arc, Vec>)>, + // Last field on purpose: struct fields drop in declaration order, so the region + // outlives every receiver handle into it. + _region: HeapRegion, + } + + fn bootstrap_mesh(n_procs: u32) -> Bootstrap { + bootstrap_mesh_with_queue(n_procs, IN_PROCESS_QUEUE_BYTES) + } + + fn bootstrap_mesh_with_queue(n_procs: u32, queue_bytes: usize) -> Bootstrap { + let region_total = dsm_region_bytes(n_procs, queue_bytes, 0).unwrap(); + let region = HeapRegion::new(region_total); + let base = SharedBase(region.base()); + let wakeup: Arc = Arc::new(NoopWakeup); + let leader_mesh = unsafe { + leader_setup( + base.0, + n_procs, + queue_bytes, + &[], + Arc::clone(&wakeup), + receiver_token(0), + Arc::new(NoInterrupt), + /* attach_senders */ false, + ) + } + .unwrap() + .mesh; + let mut workers = Vec::new(); + for proc_idx in 1..n_procs { + let attach = unsafe { + worker_setup( + base.0, + region_total, + proc_idx, + Arc::clone(&wakeup), + receiver_token(proc_idx), + Arc::new(NoInterrupt), + ) + } + .unwrap(); + workers.push((proc_idx, attach.mesh, attach.outbound_senders)); + } + Bootstrap { + leader_mesh, + workers, + _region: region, + } + } + + /// A `GROUP BY` plans a nested `NetworkShuffleExec`, so this exercises hash-routed + /// worker-to-worker traffic and the self-loop sender, which the plain gather test never + /// touches. That routing is the main thing an upstream rebase can silently break. + #[tokio::test(flavor = "current_thread")] + async fn in_process_shuffle_query_matches_serial() { + let query = "SELECT val, count(*) AS c FROM t GROUP BY val ORDER BY val"; + + let serial_ctx = SessionContext::new(); + register_table(&serial_ctx); + let expected = serial_ctx + .sql(query) + .await + .unwrap() + .collect() + .await + .unwrap(); + + let boot = bootstrap_mesh(N_WORKERS + 1); + let captured = new_captured_plans(); + let leader_ctx = build_session(Arc::clone(&boot.leader_mesh), Some(Arc::clone(&captured))); + let physical = leader_ctx + .sql(query) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let entries = collect_dispatched_stages(&physical, N_WORKERS); + assert!( + entries + .iter() + .any(|e| matches!(e.routing, FragmentRouting::Hashed { .. })), + "expected a hash-routed producer stage; got {:?}", + entries.iter().map(|e| &e.routing).collect::>() + ); + + let mut workers = JoinSet::new(); + for (proc_idx, mesh, outbound) in boot.workers { + let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); + let session = build_session(Arc::clone(&mesh), None); + workers.spawn(run_worker_proc( + fragments, + outbound, + mesh, + session, + N_WORKERS, + Arc::clone(&captured), + )); + } + + let leader_task_ctx = leader_ctx.task_ctx(); + let stream = physical.execute(0, leader_task_ctx).unwrap(); + let got: Vec = stream.try_collect().await.unwrap(); + + while let Some(res) = workers.join_next().await { + res.expect("worker task panicked").expect("worker proc"); + } + + use datafusion::arrow::util::pretty::pretty_format_batches; + assert_eq!( + pretty_format_batches(&expected).unwrap().to_string(), + pretty_format_batches(&got).unwrap().to_string(), + "distributed shuffle != serial" + ); + } + + /// An aggregate emits offset slices of its accumulated state, and arrow-ipc writes a + /// sliced variable-length array's whole values buffer, so raw frames balloon to the + /// state's size regardless of batch size. With rings far smaller than that state, the + /// send path's compact-and-chunk has to carry the query; the result must still match the + /// serial reference exactly. + #[tokio::test(flavor = "current_thread")] + async fn oversized_aggregate_frames_stream_across_tiny_rings() { + let query = "SELECT val, max(s) AS m FROM t GROUP BY val ORDER BY val"; + + let serial_ctx = SessionContext::new(); + register_wide_table(&serial_ctx); + let expected = serial_ctx + .sql(query) + .await + .unwrap() + .collect() + .await + .unwrap(); + + // 64 KiB rings against ~700 KiB of per-worker aggregate state. + let boot = bootstrap_mesh_with_queue(N_WORKERS + 1, 64 * 1024); + let captured = new_captured_plans(); + let leader_ctx = build_session(Arc::clone(&boot.leader_mesh), Some(Arc::clone(&captured))); + register_wide_table(&leader_ctx); + let physical = leader_ctx + .sql(query) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let entries = collect_dispatched_stages(&physical, N_WORKERS); + + let mut workers = JoinSet::new(); + for (proc_idx, mesh, outbound) in boot.workers { + let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); + let session = build_session(Arc::clone(&mesh), None); + workers.spawn(run_worker_proc( + fragments, + outbound, + mesh, + session, + N_WORKERS, + Arc::clone(&captured), + )); + } + + let leader_task_ctx = leader_ctx.task_ctx(); + let stream = physical.execute(0, leader_task_ctx).unwrap(); + let got: Vec = stream.try_collect().await.unwrap(); + + while let Some(res) = workers.join_next().await { + res.expect("worker task panicked").expect("worker proc"); + } + + use datafusion::arrow::util::pretty::pretty_format_batches; + assert_eq!( + pretty_format_batches(&expected).unwrap().to_string(), + pretty_format_batches(&got).unwrap().to_string(), + "distributed wide aggregate != serial" + ); + } + + /// A producer that attaches and then goes away without sending its EOFs must fail the + /// gather, not hang it: the drain fails the channels the dead receiver fed once the ring + /// detaches. + #[tokio::test(flavor = "current_thread")] + async fn producer_loss_fails_the_gather_instead_of_hanging() { + let query = "SELECT id, val FROM t ORDER BY id"; + + let boot = bootstrap_mesh(N_WORKERS + 1); + let captured = new_captured_plans(); + let leader_ctx = build_session(Arc::clone(&boot.leader_mesh), Some(Arc::clone(&captured))); + let physical = leader_ctx + .sql(query) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + let entries = collect_dispatched_stages(&physical, N_WORKERS); + + let mut workers = JoinSet::new(); + for (proc_idx, mesh, outbound) in boot.workers { + if proc_idx == 1 { + // Simulated crash: the proc attached (its senders exist), then dies without + // running its fragments or sending EOF. Dropping the senders is what process + // exit does. + drop(outbound); + drop(mesh); + continue; + } + let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); + let session = build_session(Arc::clone(&mesh), None); + workers.spawn(run_worker_proc( + fragments, + outbound, + mesh, + session, + N_WORKERS, + Arc::clone(&captured), + )); + } + + let leader_task_ctx = leader_ctx.task_ctx(); + let stream = physical.execute(0, leader_task_ctx).unwrap(); + let res: Result, _> = stream.try_collect().await; + + while let Some(r) = workers.join_next().await { + r.expect("worker task panicked").expect("worker proc"); + } + + let err = res + .expect_err("gather must fail when a producer goes away") + .to_string(); + assert!( + err.contains("detached before this channel's EOF"), + "unexpected error: {err}" + ); + } +} diff --git a/src/shm/mesh.rs b/src/shm/mesh.rs new file mode 100644 index 000000000..90391d53a --- /dev/null +++ b/src/shm/mesh.rs @@ -0,0 +1,379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! MPP mesh adapters around `DsmMpscRing` plus the alignment helpers used to size +//! the per-inbox DSM regions. + +use datafusion::common::DataFusionError; + +use super::mpsc_ring::{ + DsmMpscReceiver, DsmMpscSender, RecvOutcome as MpscRecvOutcome, SendError as MpscSendError, +}; +use super::transport::{BatchChannelReceiver, BatchChannelSender, RecvOutcome}; + +/// Postgres MAXALIGN. The DSM layout must agree between this crate (which computes the per-inbox +/// offsets) and whatever shared buffer the embedder hands it. PG's `MAXIMUM_ALIGNOF` is 8 on every +/// supported platform, so the shared-memory layout pins it. +const MAXALIGN: usize = 8; + +/// MAXALIGN-DOWN of `queue_bytes`, so every per-inbox region in the grid is the same size and +/// offset math is a simple multiply. +#[inline] +pub(super) fn aligned_queue_bytes(queue_bytes: usize) -> usize { + queue_bytes & !(MAXALIGN - 1) +} + +/// MAXALIGN-UP `n` to the next multiple of [`MAXALIGN`]. Returns `None` on overflow. Used by +/// [`super::dsm::compute_dsm_layout`] to keep section boundaries MAXALIGN-aligned. +#[inline] +pub(super) fn align_up_maxalign_checked(n: usize) -> Option { + let mask = MAXALIGN - 1; + n.checked_add(mask).map(|x| x & !mask) +} + +/// DSM MPSC ring as a `BatchChannelSender`. Multiple producer processes hold their own +/// `DsmInboxSender` clones targeting the same receiver inbox; the ring serializes them +/// via Vyukov CAS on `tail`. Because the ring's atomics are the synchronization point, +/// the `send_bytes` / `try_send_bytes` paths don't need an attach-thread `debug_assert!`. +/// +/// Detach-on-drop: `DsmMpscSender::Drop` decrements `sender_count`; the last drop flips +/// `detached` and wakes the receiver, mirroring shm_mq's "drop the last sender, receiver +/// sees detach" guarantee. +pub(super) struct DsmInboxSender { + inner: DsmMpscSender, + send_lock: tokio::sync::Mutex<()>, +} + +// Send + Sync auto-derive. Both fields are Send + Sync (`DsmMpscSender` via its unsafe +// impls; `tokio::sync::Mutex` by definition), so no manual `unsafe impl` is needed; the +// auto-derive also surfaces a compile error if a future field is `!Send` / `!Sync`. + +impl DsmInboxSender { + /// Wrap a `DsmMpscSender` for use through the `BatchChannelSender` trait. + pub(super) fn new(inner: DsmMpscSender) -> Self { + Self { + inner, + send_lock: tokio::sync::Mutex::new(()), + } + } + + fn map_send_err(err: MpscSendError) -> DataFusionError { + match err { + MpscSendError::Detached => { + DataFusionError::Execution("mpp: DSM MPSC inbox detached".into()) + } + MpscSendError::MessageTooLarge => DataFusionError::Execution( + "mpp: DSM MPSC frame exceeds the entire ring capacity \ + (raise the embedder's queue-size knob)" + .into(), + ), + MpscSendError::Full => DataFusionError::Execution( + "mpp: DSM MPSC inbox full (caller should retry via try_send_bytes)".into(), + ), + } + } +} + +impl BatchChannelSender for DsmInboxSender { + fn send_bytes(&self, bytes: &[u8]) -> Result<(), DataFusionError> { + // Fallback for callers that didn't wire `with_cooperative_drain`. The real send + // path drives `try_send_bytes` through the cooperative spin in `transport.rs`; + // this loop just spins on `yield_now` and burns the backend core under a slow + // consumer. Hitting it in production means a missing `with_cooperative_drain` + // on the fragment, not a real backpressure path. + loop { + match self.inner.try_send(bytes) { + Ok(()) => return Ok(()), + Err(MpscSendError::Full) => std::thread::yield_now(), + Err(e) => return Err(Self::map_send_err(e)), + } + } + } + + fn try_send_bytes(&self, bytes: &[u8]) -> Result { + match self.inner.try_send(bytes) { + Ok(()) => Ok(true), + Err(MpscSendError::Full) => Ok(false), + Err(e) => Err(Self::map_send_err(e)), + } + } + + fn send_lock(&self) -> &tokio::sync::Mutex<()> { + &self.send_lock + } + + fn max_frame_bytes(&self) -> Option { + Some(self.inner.max_frame_bytes()) + } +} + +/// DSM MPSC ring as a `BatchChannelReceiver`. The scratch `Vec` lives behind a `Mutex` so a +/// `&self` `try_recv` can hand the populated buffer back via `mem::take` without `RefCell` runtime +/// borrow tracking. +/// +/// Single-consumer comes from the call pattern: one `DsmInboxReceiver` per process, +/// owned by `DrainHandle::cooperative_receivers`, polled inline from `try_drain_pass`. +/// No two threads ever race on the same receiver; the mutex is just interior-mutability +/// boilerplate, uncontended in production. +pub(super) struct DsmInboxReceiver { + inner: DsmMpscReceiver, + /// Scratch the inner primitive's `try_recv` reuses across calls (via reserve+set_len). + /// We `mem::take` it on every Bytes outcome to hand the bytes to the caller without a + /// copy; the inner primitive re-grows from `Vec::new()` on the next call. + scratch: std::sync::Mutex>, +} + +// `DsmMpscReceiver` is deliberately `!Sync` (single-consumer invariant). We promote +// `DsmInboxReceiver` to Sync because the caller pattern guarantees only one thread +// touches it at a time, and `Mutex>` protects the scratch from shared-reference +// UB if that invariant ever slips. Send is auto-derived. +unsafe impl Sync for DsmInboxReceiver {} + +impl DsmInboxReceiver { + /// Wrap a `DsmMpscReceiver` for use through the `BatchChannelReceiver` trait. + pub(super) fn new(inner: DsmMpscReceiver) -> Self { + Self { + inner, + scratch: std::sync::Mutex::new(Vec::new()), + } + } + + /// Register the consumer's opaque wakeup token on the underlying ring. The embedder packs + /// whatever its [`super::mpsc_ring::Wakeup`] interprets (a PG embedder packs `(pgprocno, pid)`). + pub(super) fn set_receiver(&self, token: u64) { + self.inner.set_receiver(token); + } +} + +impl BatchChannelReceiver for DsmInboxReceiver { + fn try_recv(&self) -> RecvOutcome { + let mut buf = self.scratch.lock().expect("scratch mutex poisoned"); + match self.inner.try_recv(&mut buf) { + MpscRecvOutcome::Bytes => { + // Hand the populated buffer to the caller and leave an empty Vec behind. + // The inner primitive's `try_recv` does its own `reserve(len)` on the + // next call, so we don't need to pre-allocate scratch capacity here. + RecvOutcome::Bytes(std::mem::take(&mut *buf)) + } + MpscRecvOutcome::Empty => RecvOutcome::Empty, + MpscRecvOutcome::Detached => RecvOutcome::Detached, + } + } +} + +#[cfg(test)] +mod tests { + use super::super::mpsc_ring::{self, DsmMpscRingHeader, Wakeup}; + use super::*; + + struct NoopWakeup; + impl Wakeup for NoopWakeup { + fn wake(&self, _token: u64) {} + } + fn test_wakeup() -> std::sync::Arc { + std::sync::Arc::new(NoopWakeup) + } + + /// Allocate a fresh ring (heap, aligned) and return `(owning region, sender, receiver)`. + /// Pairs `DsmInboxSender` + `DsmInboxReceiver` over a heap-allocated ring matching the + /// alignment contract `create_at` requires. Production allocates the region inside a + /// `dsm_segment`; this helper exists so the BatchChannel trait impls can be exercised + /// without a PG backend. + /// + /// The region is returned FIRST so callers bind it first. Rust drops locals in reverse + /// declaration order, so the region (bound first) drops LAST, after the sender/receiver whose + /// `Drop` touches the ring memory (`DsmMpscSender::drop` does `sender_count.fetch_sub`). + /// Returning the region last instead frees the bytes before those `Drop`s run, a + /// use-after-free that corrupts the heap shared with parallel tests. + fn test_dsm_inbox_pair( + ring_size: u32, + slot_capacity: u32, + ) -> (AlignedTestRegion, DsmInboxSender, DsmInboxReceiver) { + let bytes = DsmMpscRingHeader::region_bytes(ring_size, slot_capacity); + let region = AlignedTestRegion::new(bytes); + let header_ptr = + unsafe { mpsc_ring::create_at(region.as_mut_ptr(), ring_size, slot_capacity) }; + let nn = std::ptr::NonNull::new(header_ptr).expect("create_at returned null"); + let alive = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let sender = DsmInboxSender::new(unsafe { + DsmMpscSender::new(nn, test_wakeup(), std::sync::Arc::clone(&alive)) + }); + let receiver = DsmInboxReceiver::new(unsafe { DsmMpscReceiver::new(nn, alive) }); + (region, sender, receiver) + } + + struct AlignedTestRegion { + ptr: *mut u8, + layout: std::alloc::Layout, + } + + impl AlignedTestRegion { + fn new(bytes: usize) -> Self { + let align = std::mem::align_of::(); + let layout = std::alloc::Layout::from_size_align(bytes, align).expect("layout"); + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null()); + Self { ptr, layout } + } + fn as_mut_ptr(&self) -> *mut u8 { + self.ptr + } + } + + impl Drop for AlignedTestRegion { + fn drop(&mut self) { + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } + } + + #[test] + fn aligned_queue_bytes_rounds_down_to_maxalign() { + let maxalign = MAXALIGN; + for req in [0, 1, 7, 8, 15, 16, 1024, 1025, 1_048_576, 8 * 1024 * 1024] { + let got = aligned_queue_bytes(req); + assert!(got <= req); + assert_eq!(got % maxalign, 0, "not aligned: {got} for req {req}"); + assert!(req - got < maxalign); + } + } + + #[test] + fn align_up_maxalign_rounds_up_to_maxalign() { + let maxalign = MAXALIGN; + for req in [0, 1, 7, 8, 15, 16, 1024, 1025] { + let got = align_up_maxalign_checked(req).unwrap(); + assert!(got >= req); + assert_eq!(got % maxalign, 0); + assert!(got - req < maxalign); + } + assert!(align_up_maxalign_checked(usize::MAX).is_none()); + } + + #[test] + fn dsm_inbox_batch_channel_round_trip() { + let (_region, tx, rx) = test_dsm_inbox_pair(4, 64); + // Sanity: try_send_bytes succeeds, try_recv hands back the bytes, send_lock + // returns a usable Mutex. + assert!(tx.try_send_bytes(b"hello").unwrap()); + assert!(tx.try_send_bytes(b"world").unwrap()); + match rx.try_recv() { + RecvOutcome::Bytes(b) => assert_eq!(&b[..], b"hello"), + other => panic!("expected Bytes(hello), got {other:?}"), + } + match rx.try_recv() { + RecvOutcome::Bytes(b) => assert_eq!(&b[..], b"world"), + other => panic!("expected Bytes(world), got {other:?}"), + } + assert!(matches!(rx.try_recv(), RecvOutcome::Empty)); + // send_lock is just exercised: the per-instance Mutex satisfies the trait. + let _guard = tx.send_lock(); + } + + #[test] + fn dsm_inbox_try_send_returns_false_when_full() { + let (_region, tx, rx) = test_dsm_inbox_pair(2, 64); + assert!(tx.try_send_bytes(b"a").unwrap()); + assert!(tx.try_send_bytes(b"b").unwrap()); + // Third send should hit Full (returns Ok(false), not Err). + assert!(!tx.try_send_bytes(b"c").unwrap()); + // Drain one, then send again succeeds. + assert!(matches!(rx.try_recv(), RecvOutcome::Bytes(_))); + assert!(tx.try_send_bytes(b"c").unwrap()); + } + + #[test] + fn dsm_inbox_multi_producer_through_trait_surface() { + use std::sync::Arc; + // Build a real Arc shared across threads to confirm + // dyn dispatch + Send/Sync impls compile and behave. + let (_region, tx, rx) = test_dsm_inbox_pair(64, 32); + let tx: Arc = Arc::new(tx); + let mut handles = Vec::new(); + const K: usize = 4; + const M: u32 = 200; + for producer_id in 0..K { + let tx = Arc::clone(&tx); + handles.push(std::thread::spawn(move || { + let mut payload = [0u8; 8]; + payload[0..4].copy_from_slice(&(producer_id as u32).to_le_bytes()); + let mut sent = 0u32; + while sent < M { + payload[4..8].copy_from_slice(&sent.to_le_bytes()); + match tx.try_send_bytes(&payload) { + Ok(true) => sent += 1, + Ok(false) => std::thread::yield_now(), + Err(e) => panic!("send failed: {e}"), + } + } + })); + } + let target = K * M as usize; + let mut seen = vec![vec![false; M as usize]; K]; + let mut got = 0usize; + while got < target { + match rx.try_recv() { + RecvOutcome::Bytes(b) => { + let producer_id = u32::from_le_bytes(b[0..4].try_into().unwrap()) as usize; + let idx = u32::from_le_bytes(b[4..8].try_into().unwrap()) as usize; + let already = std::mem::replace(&mut seen[producer_id][idx], true); + assert!(!already, "dup ({producer_id}, {idx})"); + got += 1; + } + RecvOutcome::Empty => std::thread::yield_now(), + RecvOutcome::Detached => panic!("unexpected detach"), + } + } + for h in handles { + h.join().unwrap(); + } + } + + /// Dropping the last `DsmInboxSender` flips `detached` and the receiver sees the + /// queued bytes followed by `Detached`. This is the structural equivalent of + /// shm_mq's "drop the last sender, receiver sees detach" guarantee, and is what + /// keeps the drain loop from wedging on a clean shutdown. + #[test] + fn dropping_last_sender_triggers_detach() { + let (_region, tx, rx) = test_dsm_inbox_pair(4, 64); + tx.try_send_bytes(b"final").unwrap(); + drop(tx); + // The queued frame is still readable. + match rx.try_recv() { + RecvOutcome::Bytes(b) => assert_eq!(&b[..], b"final"), + other => panic!("expected Bytes(final), got {other:?}"), + } + // Then Detached. + assert!(matches!(rx.try_recv(), RecvOutcome::Detached)); + } + + #[test] + fn try_send_bytes_rejects_oversize_payload() { + // Multi-slot fragmentation lifted the per-slot ceiling; only payloads larger + // than `ring_size * (slot_capacity - SLOT_HEADER_BYTES)` are now rejected. + // ring_size=2, slot_capacity=32 -> payload_cap=16, ring-wide cap=32. 64 + // bytes still doesn't fit. + let (_region, tx, _rx) = test_dsm_inbox_pair(2, 32); + let oversize = vec![0u8; 64]; + let err = tx + .try_send_bytes(&oversize) + .expect_err("expected MessageTooLarge"); + assert!( + format!("{err}").contains("exceeds the entire ring capacity"), + "unexpected error: {err}" + ); + } +} diff --git a/src/shm/mod.rs b/src/shm/mod.rs new file mode 100644 index 000000000..bf5c95e17 --- /dev/null +++ b/src/shm/mod.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared-memory transport. +//! +//! A non-gRPC [`ChannelResolver`] for co-located execution, where "workers" are tasks or +//! parallel processes sharing one machine and communicating over a shared-memory mesh rather +//! than gRPC. The transport-mechanism pieces (the MPSC ring, framing, routing, cooperative +//! drain) live here as a reusable library; an embedder supplies the two platform primitives via +//! small extension points: how to allocate the shared buffer, and how to wake a blocked consumer. +//! +//! The point of hosting it in this crate is testing: the in-process instantiation runs real +//! distributed queries through the transport in this crate's CI, so an upstream rebase that +//! breaks the channel-protocol contract fails here, before any downstream embedder rebuilds. +//! +//! [`ChannelResolver`]: crate::ChannelResolver +//! +//! Two assumptions an embedder signs up for: +//! - Execution is cooperative on a current-thread runtime: consumers spin on +//! `try_pop` + `yield_now` and producers drain their own inbound while blocked, instead of +//! parking on the `Wakeup` extension point. On a multi-thread runtime each stream burns a core while +//! idle. +//! - Inbound frames demux into unbounded per-channel buffers, so a consumer that falls behind +//! buffers the in-flight intermediate result in process memory. The rings in shared memory +//! stay bounded; the overflow lives on the consumer's heap. + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +mod dsm; +mod mesh; +mod mpsc_ring; +mod runtime; +// Deferred: the self-hosting default transport was built on the removed `WorkerTransport`/ +// `WorkerDispatch` dispatch umbrella, which the `ChannelResolver` model has no analog for; its +// no-gRPC-default role is now served by `InProcessChannelResolver` and its ring-exercising role by +// the `in_process` test, so it stays gated out until reimplemented on `coordinator_channel`. +#[cfg(any())] +mod self_hosted; +mod setup; +mod sink; +mod transport; + +// Curated public surface an embedder consumes. The embedder allocates the shared buffer and +// supplies the two extension points (`Wakeup`, `Interrupt`); everything else is built here. +pub use mpsc_ring::{NO_RECEIVER_TOKEN, Wakeup}; +pub use runtime::{InProcessWorkerResolver, MppMesh, ShmChannelResolver, proc_for_task}; +pub use setup::{ + LeaderAttach, WorkerAttach, collect_task_metrics, dsm_region_bytes, install_work_unit_channels, + leader_setup, region_total, run_worker_fragment, worker_setup, +}; +pub use sink::{PartitionSink, WorkerSink}; +pub use transport::{ + CooperativeDrainSet, Interrupt, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, + SendBatchStats, SetPlanFrame, +}; + +/// Out-of-DSM liveness flag shared by the ring handles from one attach. The embedder flips it to +/// `false` from its dsm-detach callback while the segment is still mapped, so a handle dropped +/// afterward (e.g. by a memory-context reset) no-ops instead of dereferencing freed memory. +pub type AliveFlag = Arc; + +// In-process instantiation + the end-to-end test that runs a real distributed query through the +// transport with no Postgres. Test-only: it's how an upstream rebase that breaks the transport +// contract fails in this crate's CI. +#[cfg(test)] +mod in_process; diff --git a/src/shm/mpsc_ring.rs b/src/shm/mpsc_ring.rs new file mode 100644 index 000000000..93e13d7e2 --- /dev/null +++ b/src/shm/mpsc_ring.rs @@ -0,0 +1,1560 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! DSM-backed MPSC ring for MPP mesh inboxes. +//! +//! PG's `shm_mq` is hard-wired SPSC (asserted in `shm_mq_set_sender`), so one inbox +//! can't be shared across N-1 peers without forking PG. This ring is the replacement: +//! a fixed-size byte-message ring sitting in a `dsm_segment`, MPSC-correct via +//! Vyukov-style per-slot sequence counters. +//! +//! Layout (contiguous bytes, `repr(C)`): +//! +//! ```text +//! +- DsmMpscRingHeader -----------------------+ +//! | magic, version, ring_size, slot_capacity | +//! | sender_count, detached, receiver_packed | +//! | (cache-line padding around head/tail) | +//! | head, tail (each on its own line) | +//! +-------------------------------------------+ +//! | Slot[0] | Slot[1] | ... | Slot[N-1] | +//! +-------------------------------------------+ +//! +//! Slot { +//! seq: AtomicU64, // Vyukov phase counter +//! len: AtomicU32, +//! data: [u8; slot_capacity - SLOT_HEADER_SIZE] +//! } +//! ``` +//! +//! Slot phase encoding (Vyukov MPMC reduced to MPSC): +//! +//! ```text +//! slot[i] in round k: seq = k * ring_size + i // empty +//! seq = k * ring_size + i + 1 // ready +//! ``` +//! +//! Producer claim at `tail = T`: read `slot[T % ring_size].seq`. `seq == T` then CAS +//! `tail: T → T+1`, winner copies payload and stores `seq = T + 1` (Release). `seq < T` +//! means ring full. `seq > T` means another producer took `T`, retry. Winner +//! `SetLatch`es the receiver. +//! +//! Consumer take at `head = H`: `slot[H % ring_size].seq == H + 1` means ready. Read +//! payload, store `seq = H + ring_size` (next round's empty marker), then `head = H+1`. +//! The single consumer owns `head` without CAS; producers contend only on `tail`. +//! +//! **Safety**: public methods on `DsmMpscSender` / `DsmMpscReceiver` are type-safe once +//! constructed. The constructors are `unsafe` because the DSM region must be correctly +//! sized and not aliased (one process calls `create_at`, everyone else `attach_at`). +//! +//! **Counter wraparound**: head/tail are `u64`, incremented by one per op. At 100M +//! ops/sec that's ~5800 years, so we ignore wrap. The seq math would break under wrap +//! (pre-wrap `seq` would exceed post-wrap `tail` and producers would spin forever); if +//! that ever matters, add a `tail < u64::MAX - margin` check and reset the ring. + +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; + +use std::sync::Arc; + +use super::AliveFlag; + +/// Wakes the ring's single consumer after a producer publishes a frame. +/// +/// The ring is transport-agnostic: the consumer registers an opaque `u64` token via +/// `DsmMpscReceiver::set_receiver`, and each publishing producer hands that token to this +/// hook. An embedder over PostgreSQL shared memory packs `(pgprocno, pid)` into the token and +/// `SetLatch`es the backend; an in-process embedder packs a registry key and unparks the +/// consumer thread. The token is stored in the ring header (shared memory), so the hook must be +/// able to resolve it from any producer. +pub trait Wakeup: Send + Sync { + fn wake(&self, token: u64); +} + +/// Sentinel token meaning "no consumer registered". Producers skip the wake when the stored +/// token equals this; `create_at` initializes the ring to it. +pub const NO_RECEIVER_TOKEN: u64 = u64::MAX; + +/// Reserved byte at the start of every slot. Sized so payload + header fits in +/// `slot_capacity` exactly. +const SLOT_HEADER_BYTES: usize = std::mem::size_of::(); + +#[repr(C)] +struct SlotHeader { + /// Vyukov sequence counter; see module docs for phase encoding. + seq: AtomicU64, + /// Bytes of payload written into THIS slot's data region; + /// `0..=slot_capacity - SLOT_HEADER_BYTES`. + len: AtomicU32, + /// Fragment metadata. Low 2 bits = [`FragmentKind`] (Complete=0 / First=1 / + /// Continue=2 / Last=3). For `First`, bits 16..32 hold the slot count of the + /// logical frame (1..=65535). For other kinds these bits are unused. + flags: AtomicU32, +} + +/// Kind bits stored in [`SlotHeader::flags`]'s low 2 bits. +/// +/// A frame fitting in `slot_capacity - SLOT_HEADER_BYTES` rides the single-slot +/// fast path as `Complete`. Anything larger spans +/// `n_slots = ceil(frame_len / per_slot_payload)` consecutive slots: `First` +/// (carrying `n_slots` in the upper bits), `Continue` for the middle, `Last` for +/// the tail. Producers grab the whole run with one +/// `tail.compare_exchange(T, T + n_slots)`, so other producers can't interleave +/// their fragments and the receiver always sees the slots in producer order. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FragmentKind { + Complete = 0, + First = 1, + Continue = 2, + Last = 3, +} + +/// Bit-mask for the kind bits in [`SlotHeader::flags`]. +const FLAGS_KIND_MASK: u32 = 0b11; +/// Shift for the `n_slots` field stored in the flags' upper half (only meaningful +/// for `First`). Limits per-frame fragmentation to 65535 slots, which is far +/// beyond any realistic ring size. +const FLAGS_NSLOTS_SHIFT: u32 = 16; +const FLAGS_NSLOTS_MAX: u32 = 0xFFFF; + +#[inline] +fn pack_flags(kind: FragmentKind, n_slots: u32) -> u32 { + debug_assert!( + kind != FragmentKind::First || (1..=FLAGS_NSLOTS_MAX).contains(&n_slots), + "First frames must carry 1..=65535 slots; got {n_slots}" + ); + (kind as u32) | (n_slots << FLAGS_NSLOTS_SHIFT) +} + +#[inline] +fn unpack_kind(flags: u32) -> Option { + match flags & FLAGS_KIND_MASK { + 0 => Some(FragmentKind::Complete), + 1 => Some(FragmentKind::First), + 2 => Some(FragmentKind::Continue), + 3 => Some(FragmentKind::Last), + _ => None, + } +} + +#[inline] +fn unpack_nslots(flags: u32) -> u32 { + flags >> FLAGS_NSLOTS_SHIFT +} + +/// Magic constant validating that an attaching process points at a `DsmMpscRingHeader` +/// rather than garbage. "MPCR" = MPSC Ring. Different value from `MppDsmHeader`'s magic +/// so a worker that picks up the wrong region fails the wrong-shape check loudly. +const MPSC_RING_MAGIC: u32 = u32::from_le_bytes(*b"MPCR"); + +/// Bump on any wire-incompatible layout change. Mirrors the discipline at +/// `MppDsmHeader::validate`. +/// +/// Wire versions: +/// - v1: `SlotHeader { seq, len, _pad }`. One frame per slot. +/// - v2: `SlotHeader { seq, len, flags }`. `flags` carries `FragmentKind` plus +/// `n_slots` on `First`, so frames bigger than +/// `slot_capacity - SLOT_HEADER_BYTES` can span N consecutive slots reserved +/// atomically by the producer. +const MPSC_RING_VERSION: u32 = 2; + +/// Assumed cache line size for false-sharing avoidance. 64 bytes covers x86_64 and arm64; +/// over-padding on smaller-cache-line targets costs a few bytes per ring, nothing more. +const CACHE_LINE: usize = 64; + +/// Ring header. Laid out first in the DSM region; slot array follows immediately after. +/// +/// Cache-line padding around `head` and `tail` isn't optional: at N=24 producer +/// contention the consumer's `head.store` and the producers' `tail.compare_exchange` +/// race on the same line, MESI-ping-ponging every claim. That's the false-sharing +/// footgun Vyukov's writeups call out; the Disruptor literature shows 5-10x throughput +/// loss from it on x86. Padding puts each hot field on its own line. +/// +/// NOT `#[repr(C, align(64))]`: PG `dsm_segment` base addresses are only +/// MAXALIGN-aligned in practice (on macOS the user-data offset can land 16-aligned +/// but not 64-aligned). Forcing `align(64)` would impose a 64-aligned destination on +/// `create_at` / `attach_at` we can't guarantee. The `_pad_*` fields below still +/// put `head`, `tail`, and the first slot on separate 64-byte regions; it's the +/// *distance* between hot fields that matters, not their absolute alignment. +#[repr(C)] +pub(super) struct DsmMpscRingHeader { + /// Magic constant; equals `MPSC_RING_MAGIC` for a valid ring. Checked in `attach_at`. + magic: u32, + /// Layout version; equals `MPSC_RING_VERSION`. Checked in `attach_at`. + version: u32, + /// Number of slots. Immutable after `create_at`. + ring_size: u32, + /// Byte capacity of each slot INCLUDING the slot header. Payload bytes per slot are + /// `slot_capacity - SLOT_HEADER_BYTES`. Immutable after `create_at`. + slot_capacity: u32, + /// Live `DsmMpscSender` count. Incremented in `DsmMpscSender::new`, decremented in + /// `DsmMpscSender::Drop`. The drop that takes the count from 1 → 0 sets `detached` + /// (with Release) and wakes the receiver, mirroring shm_mq's "drop = detach" + /// structural guarantee. + sender_count: AtomicU32, + /// Set by the consumer (or by the leader on query teardown) to tell producers to + /// fail-fast on subsequent sends. Sticky. + detached: AtomicBool, + _pad_after_detached: [u8; 3], + /// Packed `(pgprocno: i32, pid: i32)` of the registered receiver, or 0 (both + /// Opaque receiver token, set by the consumer via `set_receiver` and handed to the + /// embedder's `Wakeup` extension point on every post-publish wake. Initialized to + /// [`NO_RECEIVER_TOKEN`] (`u64::MAX`), which producers treat as "no receiver yet: + /// skip the wake". The embedder defines the token's contents (pg_search packs + /// `(pgprocno, pid)`); a single atomic keeps whatever pair it packs from being + /// observed torn mid-update. + receiver_packed: AtomicU64, + /// Padding to push `head` onto its own cache line. Header up to here uses bytes + /// 0..32; this padding fills 32..64 so `head` lands at offset 64 exactly. The + /// `header_layout_is_cache_friendly` test asserts this. + _pad_before_head: [u8; CACHE_LINE - 32], + /// Consumer's read cursor. Only the consumer writes this. Currently no producer + /// reads it (full-detection works via slot `seq`); the Release-on-store is defensive + /// for any future blocking-send variant that wants to poll consumer progress. + head: AtomicU64, + /// Padding to push `tail` onto its own cache line so the consumer's `head.store` + /// doesn't invalidate the producers' `tail` cache line. + _pad_between_head_and_tail: [u8; CACHE_LINE - 8], + /// Producers' write cursor. CAS'd to claim slot ownership for a tail value. + tail: AtomicU64, + /// Padding so the first slot doesn't share a cache line with `tail`. Producers + /// race on `tail`; the consumer's first slot read should not pull the `tail` cache + /// line into the consumer's L1 unnecessarily. + _pad_after_tail: [u8; CACHE_LINE - 8], +} + +impl DsmMpscRingHeader { + /// Bytes occupied by `ring_size` slots of `slot_capacity` each, plus the header. + pub(super) const fn region_bytes(ring_size: u32, slot_capacity: u32) -> usize { + std::mem::size_of::() + (ring_size as usize) * (slot_capacity as usize) + } +} + +// The receiver wake lives on `DsmMpscSender` (it holds the injected `Wakeup`); see +// `DsmMpscSender::wake_receiver`. + +/// Errors that `try_send` can surface to the producer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum SendError { + /// Ring is full; consumer hasn't drained enough to free a slot. + Full, + /// Receiver has detached (query teardown). Producer should stop sending. + Detached, + /// `bytes.len() + SLOT_HEADER_BYTES > slot_capacity`. The caller picked a slot + /// capacity too small for this payload; bumping `slot_capacity` at `create_at` time + /// fixes it. + MessageTooLarge, +} + +/// Outcome of a single `try_recv` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RecvOutcome { + /// A frame was copied into the caller's buffer. + Bytes, + /// Ring is empty right now; try again later. + Empty, + /// All producers have detached and there's no more data to drain. + Detached, +} + +/// Caller-visible handle for the single consumer. +pub(super) struct DsmMpscReceiver { + ring: NonNull, + alive: AliveFlag, +} + +/// Caller-visible handle for any of the N-1 producers. +pub(super) struct DsmMpscSender { + ring: NonNull, + /// Hook to wake the registered consumer after a publish. Injected by the embedder so the + /// ring stays free of any process/thread-notification mechanism. + wakeup: Arc, + /// Whether this handle counts toward `sender_count`. Data senders (the producer fragments) + /// do, so the last one's drop flips `detached` and the consumer learns its producers are gone. + /// Control senders (a consumer's `Cancel` path) don't: they target a peer's inbox without being + /// one of its producers, so counting them would mask that peer's own producer-gone signal. + counts_as_data: bool, + alive: AliveFlag, +} + +// SAFETY: the ring is a `repr(C)` blob in shared memory whose atomic operations are the +// synchronization point. Both handles are stateless pointers to the same data; sending +// either across threads requires only the atomic ordering already in use. +// +// `DsmMpscReceiver` is deliberately !Sync: the type-level invariant that exactly one +// thread calls `try_recv` at a time is what makes the lock-free MPSC math correct (the +// single consumer owns `head` without CAS). `DsmMpscSender` is Sync so multiple producer +// threads can share one `Arc` and race on `tail` via CAS. +unsafe impl Send for DsmMpscReceiver {} +unsafe impl Send for DsmMpscSender {} +unsafe impl Sync for DsmMpscSender {} + +/// Initialize a freshly-allocated DSM region into a valid ring header + zeroed slot array. +/// Must be called exactly once, by the process that allocated the region (leader in our +/// case). All other processes attach via [`attach_at`] without re-initializing. +/// +/// # Safety +/// - `base` must point at the start of a region of at least +/// `DsmMpscRingHeader::region_bytes(ring_size, slot_capacity)` bytes. +/// - `ring_size >= 2` (a ring of 1 slot can't distinguish empty from full). +/// - `slot_capacity > SLOT_HEADER_BYTES` (need at least one byte of payload). +/// - The region must not be concurrently accessed by any other process or thread until +/// this returns. +pub(super) unsafe fn create_at( + base: *mut u8, + ring_size: u32, + slot_capacity: u32, +) -> *mut DsmMpscRingHeader { + debug_assert!(ring_size >= 2, "ring_size must be >= 2"); + debug_assert!( + slot_capacity as usize > SLOT_HEADER_BYTES, + "slot_capacity must leave room for at least one payload byte" + ); + // The header's natural alignment (from its largest field, AtomicU64) is 8 bytes. + // PG MAXALIGN is 8 on every supported platform, so dsm_segment user-data offsets + // computed via `align_up_maxalign_checked` always satisfy this. Tests use an + // aligned-Vec helper just to keep the assertion clean across allocators. + debug_assert!( + (base as usize).is_multiple_of(std::mem::align_of::()), + "create_at base must be aligned to {} bytes", + std::mem::align_of::() + ); + let header_ptr = base.cast::(); + // Write the immutable header fields. Use std::ptr::write so we don't construct an + // intermediate &mut that aliases the not-yet-initialized atomic fields. + unsafe { + std::ptr::write( + header_ptr, + DsmMpscRingHeader { + magic: MPSC_RING_MAGIC, + version: MPSC_RING_VERSION, + ring_size, + slot_capacity, + detached: AtomicBool::new(false), + _pad_after_detached: [0; 3], + sender_count: AtomicU32::new(0), + receiver_packed: AtomicU64::new(NO_RECEIVER_TOKEN), + _pad_before_head: [0; CACHE_LINE - 32], + head: AtomicU64::new(0), + _pad_between_head_and_tail: [0; CACHE_LINE - 8], + tail: AtomicU64::new(0), + _pad_after_tail: [0; CACHE_LINE - 8], + }, + ); + } + // Initialize slot sequences: slot[i].seq = i in round 0. + for i in 0..ring_size { + let slot = unsafe { slot_ptr(header_ptr, i, slot_capacity) }; + unsafe { + std::ptr::write( + slot, + SlotHeader { + seq: AtomicU64::new(i as u64), + len: AtomicU32::new(0), + flags: AtomicU32::new(0), + }, + ); + } + } + header_ptr +} + +/// Take an already-initialized ring header pointer and confirm its shape matches caller's +/// expectations. The caller's `expected_ring_size` / `expected_slot_capacity` must match +/// the values written at `create_at` time; mismatch is a hard error (returns null). +/// +/// # Safety +/// - `base` must point at the same region a previous `create_at` initialized. +/// - The region must not be deallocated for the lifetime of any handle returned from +/// the wrappers (`DsmMpscReceiver::new`, `DsmMpscSender::new`). +pub(super) unsafe fn attach_at( + base: *mut u8, + expected_ring_size: u32, + expected_slot_capacity: u32, +) -> Option> { + let header_ptr = base.cast::(); + let nn = NonNull::new(header_ptr)?; + let header = unsafe { nn.as_ref() }; + if header.magic != MPSC_RING_MAGIC || header.version != MPSC_RING_VERSION { + return None; + } + if header.ring_size != expected_ring_size || header.slot_capacity != expected_slot_capacity { + return None; + } + Some(nn) +} + +#[inline] +unsafe fn slot_ptr( + header: *mut DsmMpscRingHeader, + idx: u32, + slot_capacity: u32, +) -> *mut SlotHeader { + let header_bytes = std::mem::size_of::(); + let base = header.cast::(); + unsafe { base.add(header_bytes + (idx as usize) * (slot_capacity as usize)) } + .cast::() +} + +#[inline] +unsafe fn slot_data_ptr(slot: *mut SlotHeader) -> *mut u8 { + unsafe { slot.cast::().add(SLOT_HEADER_BYTES) } +} + +impl DsmMpscReceiver { + /// Wrap an already-initialized ring as the single consumer. Pairs with + /// [`DsmMpscSender::new`] on the producer side; calling code is responsible for + /// keeping exactly one `DsmMpscReceiver` per ring. + /// + /// # Safety + /// `ring` must point to a header initialized by [`create_at`] and not yet + /// deallocated. The caller guarantees no other `DsmMpscReceiver` exists for the + /// same ring (single-consumer invariant). + pub(super) unsafe fn new(ring: NonNull, alive: AliveFlag) -> Self { + Self { ring, alive } + } + + /// Register the consumer's opaque wakeup token. Producers Acquire-load it as a single + /// `u64` (so they never see a torn token) and hand it to their [`Wakeup`] after publishing. + /// The token is the embedder's to interpret: a PG embedder packs `(pgprocno, pid)`; an + /// in-process embedder packs a registry key. Must not be [`NO_RECEIVER_TOKEN`]. + pub(super) fn set_receiver(&self, token: u64) { + if !self.alive.load(Ordering::Acquire) { + return; + } + let header = unsafe { self.ring.as_ref() }; + header.receiver_packed.store(token, Ordering::Release); + } + + /// Try to read one frame into `out`. `Bytes`: `out` holds the payload. `Empty`: + /// caller should yield and retry. `Detached`: ring drained, all producers gone, + /// no more frames coming. + /// + /// Known wedge: if a producer CAS-advances `tail` then exits/crashes before + /// publishing `seq`, the consumer sees `tail > head`, the slot's `seq` stuck at + /// the prior-round empty marker, and `detached && tail <= head` never becomes + /// true. The drain returns `Empty` forever. In production PG's parallel-worker + /// death handling bounds this (worker exit fires leader ERROR, DSM tears down), + /// but a future pass should add an explicit `PGPROC` liveness check to + /// force-detach on producer death. + pub(super) fn try_recv(&self, out: &mut Vec) -> RecvOutcome { + if !self.alive.load(Ordering::Acquire) { + // Segment detached: no DSM left to read, so report drained. + return RecvOutcome::Detached; + } + let header = unsafe { self.ring.as_ref() }; + let head = header.head.load(Ordering::Relaxed); + let slot_idx = (head % header.ring_size as u64) as u32; + let slot = unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let seq = unsafe { (*slot).seq.load(Ordering::Acquire) }; + let expected_ready = head.wrapping_add(1); + if seq != expected_ready { + // Slot not ready. Use `<=` rather than `==` so a strict invariant + // violation (tail < head, impossible under correct operation) still + // surfaces as Detached rather than wedging Empty forever. + if header.detached.load(Ordering::Acquire) + && header.tail.load(Ordering::Acquire) <= head + { + return RecvOutcome::Detached; + } + return RecvOutcome::Empty; + } + // Slot at head is ready. Inspect its kind to choose between single-slot + // fast path and multi-slot reassembly. + let flags = unsafe { (*slot).flags.load(Ordering::Relaxed) }; + let Some(kind) = unpack_kind(flags) else { + // Reserved bits set; treat as corruption. + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + }; + let payload_cap = (header.slot_capacity as usize).saturating_sub(SLOT_HEADER_BYTES); + match kind { + FragmentKind::Complete => self.recv_single_slot(header, head, slot, payload_cap, out), + FragmentKind::First => { + let n_slots = unpack_nslots(flags); + if n_slots == 0 || n_slots > header.ring_size { + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + } + self.recv_multi_slot(header, head, n_slots, payload_cap, out) + } + FragmentKind::Continue | FragmentKind::Last => { + // Encountering Continue/Last at `head` means a producer violated + // ascending-publish ordering or a previous reassembly didn't advance + // `head` past every fragment. Either is a contract break, not a + // recoverable condition; poison and detach. + header.detached.store(true, Ordering::Release); + RecvOutcome::Detached + } + } + } + + /// Read a single-slot `Complete` frame at `head` and advance. + fn recv_single_slot( + &self, + header: &DsmMpscRingHeader, + head: u64, + slot: *mut SlotHeader, + payload_cap: usize, + out: &mut Vec, + ) -> RecvOutcome { + let len_raw = unsafe { (*slot).len.load(Ordering::Relaxed) } as usize; + // Clamp against slot's payload capacity. DSM is mapped writable by every + // attached backend, so a buggy / corrupted producer could write a garbage len. + // Without this guard, `set_len + copy_nonoverlapping` would read OOB into + // neighboring slots or other DSM contents. + if len_raw > payload_cap { + // Poison the ring rather than silently returning corrupt data. + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + } + let len = len_raw; + out.clear(); + out.reserve(len); + let data = unsafe { slot_data_ptr(slot) }; + // copy_nonoverlapping before set_len so a hypothetical panic mid-copy doesn't + // leave `out` with logical-len > initialized-bytes. + unsafe { + std::ptr::copy_nonoverlapping(data, out.as_mut_ptr(), len); + out.set_len(len); + } + // Mark the slot empty for the next round. Round k empty marker is + // (k * ring_size) + slot_idx; head + ring_size is exactly that for next round. + let next_empty_seq = head.wrapping_add(header.ring_size as u64); + unsafe { (*slot).seq.store(next_empty_seq, Ordering::Release) }; + // Advance head AFTER publishing the slot's empty marker, so a producer racing + // to claim sees the empty slot before seeing the new head value. + header.head.store(head.wrapping_add(1), Ordering::Release); + RecvOutcome::Bytes + } + + /// Reassemble an `n_slots`-fragment frame at `head`. Returns `Empty` without + /// advancing `head` if any continuation slot isn't yet published (producer + /// mid-publish); the caller retries and the `First` slot stays put with its + /// flags intact. + fn recv_multi_slot( + &self, + header: &DsmMpscRingHeader, + head: u64, + n_slots: u32, + payload_cap: usize, + out: &mut Vec, + ) -> RecvOutcome { + // First pass: verify every fragment in the run is published. If not, bail + // with Empty so the caller can retry once the producer finishes. + let mut total_len: usize = 0; + for i in 0..n_slots { + let h = head.wrapping_add(i as u64); + let slot_idx = (h % header.ring_size as u64) as u32; + let slot = unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let seq = unsafe { (*slot).seq.load(Ordering::Acquire) }; + if seq != h.wrapping_add(1) { + // Continuation slot not yet ready. Don't touch `head` or any slot + // metadata; producer will publish soon, caller will retry. + return RecvOutcome::Empty; + } + let expected_kind = if i == 0 { + FragmentKind::First + } else if i + 1 == n_slots { + FragmentKind::Last + } else { + FragmentKind::Continue + }; + let flags = unsafe { (*slot).flags.load(Ordering::Relaxed) }; + if unpack_kind(flags) != Some(expected_kind) { + // Run integrity check: the producer's multi-slot publish never + // interleaves with another producer's, so the kind sequence must be + // First, Continue*, Last. Anything else is corruption. + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + } + let len = unsafe { (*slot).len.load(Ordering::Relaxed) } as usize; + if len > payload_cap || (i + 1 < n_slots && len != payload_cap) { + // Non-final fragments must be full slots (producer fills them + // first); final fragment may be partial. Anything else is + // corruption. + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + } + total_len = match total_len.checked_add(len) { + Some(v) => v, + None => { + header.detached.store(true, Ordering::Release); + return RecvOutcome::Detached; + } + }; + } + // Second pass: concatenate payloads, mark each slot empty for next round, + // advance head past the whole run in one Release store. + out.clear(); + out.reserve(total_len); + for i in 0..n_slots { + let h = head.wrapping_add(i as u64); + let slot_idx = (h % header.ring_size as u64) as u32; + let slot = unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let len = unsafe { (*slot).len.load(Ordering::Relaxed) } as usize; + let data = unsafe { slot_data_ptr(slot) }; + unsafe { + let write_at = out.as_mut_ptr().add(out.len()); + std::ptr::copy_nonoverlapping(data, write_at, len); + out.set_len(out.len() + len); + } + // Round k empty marker for slot at position h is h + ring_size. + let next_empty_seq = h.wrapping_add(header.ring_size as u64); + unsafe { (*slot).seq.store(next_empty_seq, Ordering::Release) }; + } + header + .head + .store(head.wrapping_add(n_slots as u64), Ordering::Release); + RecvOutcome::Bytes + } +} + +impl DsmMpscSender { + /// Wrap an already-initialized ring as a producer. Multiple `DsmMpscSender` + /// handles to the same ring are legal (and the point of MPSC). Increments the + /// ring's `sender_count`; the `Drop` impl decrements and, on the last drop, + /// flips `detached` and wakes the receiver. That mirrors shm_mq's + /// "drop the sender, receiver sees detach" structural guarantee. + /// + /// # Safety + /// `ring` must point to a header initialized by [`create_at`] and not yet + /// deallocated. + pub(super) unsafe fn new( + ring: NonNull, + wakeup: Arc, + alive: AliveFlag, + ) -> Self { + let header = unsafe { ring.as_ref() }; + header.sender_count.fetch_add(1, Ordering::AcqRel); + Self { + ring, + wakeup, + counts_as_data: true, + alive, + } + } + + /// A control-plane sibling onto the same ring this handle already targets: it can publish + /// frames but stays out of `sender_count`, so it never sets or delays `detached`. Used to derive + /// a proc's `Cancel` senders from its data senders without a second attach, so a consumer can + /// reach its producer without masquerading as one of that producer's data senders. + pub(super) fn to_control(&self) -> DsmMpscSender { + DsmMpscSender { + ring: self.ring, + wakeup: Arc::clone(&self.wakeup), + counts_as_data: false, + alive: Arc::clone(&self.alive), + } + } + + /// Largest frame `try_send` can accept: every slot's payload capacity combined. Senders + /// consult this before encoding, so an oversized batch can be split to fit instead of + /// erroring with `MessageTooLarge`. + pub(super) fn max_frame_bytes(&self) -> usize { + let header = unsafe { self.ring.as_ref() }; + (header.slot_capacity as usize).saturating_sub(SLOT_HEADER_BYTES) + * header.ring_size as usize + } + + /// Wake the registered consumer, if any. Reads the token the consumer stored via + /// [`DsmMpscReceiver::set_receiver`] and hands it to the injected [`Wakeup`]; skips when no + /// consumer is registered ([`NO_RECEIVER_TOKEN`]). + fn wake_receiver(&self) { + let header = unsafe { self.ring.as_ref() }; + let token = header.receiver_packed.load(Ordering::Acquire); + if token != NO_RECEIVER_TOKEN { + self.wakeup.wake(token); + } + } + + /// Push one frame onto the ring. Returns immediately: + /// - `Ok(())`: published; receiver's latch was set if installed. + /// - `Err(Full)`: no slot run available right now; caller yields + retries. + /// - `Err(Detached)`: receiver has detached; caller stops. + /// - `Err(MessageTooLarge)`: frame needs more than `ring_size` slots + /// (max writable size is `ring_size * (slot_capacity - SLOT_HEADER_BYTES)`). + /// + /// Frames up to the per-slot payload capacity take the single-slot fast path. + /// Larger frames span consecutive slots claimed atomically via one + /// `tail.compare_exchange(T, T + n_slots)`, so other producers can't + /// interleave their fragments. See [`FragmentKind`]. + pub(super) fn try_send(&self, bytes: &[u8]) -> Result<(), SendError> { + if !self.alive.load(Ordering::Acquire) { + return Err(SendError::Detached); + } + let header = unsafe { self.ring.as_ref() }; + // Control senders ignore `detached`: a `Cancel` still has to reach a producer whose own + // inbox already detached because its upstream finished while it's mid-output. + if self.counts_as_data && header.detached.load(Ordering::Acquire) { + return Err(SendError::Detached); + } + let payload_cap = (header.slot_capacity as usize).saturating_sub(SLOT_HEADER_BYTES); + if payload_cap == 0 { + return Err(SendError::MessageTooLarge); + } + // Single-slot fast path: cheaper than the multi-slot CAS dance and preserves + // the v1 hot path exactly so the no-fragmentation case takes no extra branches + // inside the claim loop. + if bytes.len() <= payload_cap { + return self.try_send_single_slot(header, bytes); + } + // Multi-slot path: fragment across `n_slots` consecutive slots. + let n_slots_usize = bytes.len().div_ceil(payload_cap); + if n_slots_usize > header.ring_size as usize || n_slots_usize > FLAGS_NSLOTS_MAX as usize { + // Frame is larger than the entire ring (or larger than the n_slots field + // can encode). The transport chunks frames under this bound before they + // reach the ring (see transport.rs), so hitting it through `MppSender` + // means a chunk-sizing bug, not an operator emitting wide state. + return Err(SendError::MessageTooLarge); + } + let n_slots = n_slots_usize as u32; + self.try_send_multi_slot(header, bytes, n_slots, payload_cap) + } + + /// Single-slot path. Identical to the v1 layout's hot path with a `flags` write + /// added; the kind is always `Complete` here. + fn try_send_single_slot( + &self, + header: &DsmMpscRingHeader, + bytes: &[u8], + ) -> Result<(), SendError> { + loop { + // Acquire load on `tail` pairs defensively with any future blocking-send + // variant that may want to observe consumer progress via `head` (we don't + // today; full-detection rides on the slot's `seq`). Cheaper to keep the + // ordering tight than to relax-then-tighten under a future audit. + let tail = header.tail.load(Ordering::Acquire); + let slot_idx = (tail % header.ring_size as u64) as u32; + let slot = unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let seq = unsafe { (*slot).seq.load(Ordering::Acquire) }; + // Three-way compare per Vyukov MPMC. + match seq.cmp(&tail) { + std::cmp::Ordering::Equal => { + // Slot is empty in our round. Try to claim by advancing tail. + // AcqRel on success so a subsequent producer's Acquire load of + // `tail` sees our claim and skips the slot we own. Relaxed on + // failure (we just retry the loop on a fresh tail load). + match header.tail.compare_exchange_weak( + tail, + tail.wrapping_add(1), + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + // We own slot[slot_idx] for tail value `tail`. + unsafe { + (*slot).len.store(bytes.len() as u32, Ordering::Relaxed); + (*slot).flags.store( + pack_flags(FragmentKind::Complete, 0), + Ordering::Relaxed, + ); + let data = slot_data_ptr(slot); + std::ptr::copy_nonoverlapping(bytes.as_ptr(), data, bytes.len()); + // Publish: ready in round k is (k * ring_size + i + 1) = tail + 1. + (*slot).seq.store(tail.wrapping_add(1), Ordering::Release); + } + // Wake the consumer (resolves the latch via pgprocno + pid). + self.wake_receiver(); + return Ok(()); + } + Err(_) => continue, // another producer took this tail; retry + } + } + std::cmp::Ordering::Less => { + // seq < tail: the consumer hasn't reclaimed slot[slot_idx] for our + // round yet. Ring is full. + return Err(SendError::Full); + } + std::cmp::Ordering::Greater => { + // seq > tail: another producer has already claimed tail. Reload and + // retry. + continue; + } + } + } + } + + /// Multi-slot path. CAS-advance `tail` from `T` to `T + n_slots` to claim the run, + /// then publish each fragment ascending with `First` / `Continue` / `Last` flags. + /// The CAS requires all N target slots to already be in their expected empty + /// round, which generalizes v1's per-slot `seq == tail` check to a run. + /// + /// Ascending publish order isn't optional: the receiver only starts reassembling + /// once `slot[head]` (the `First`) is ready, then waits for each subsequent slot. + /// Out-of-order publish would make the receiver block on a `Continue` whose data + /// is already there but whose `seq` hasn't been stored yet, wasting one drain + /// pass per fragment. + // Fairness caveat: a multi-slot frame needs `n_slots` consecutive empty slots starting at + // `tail`, and competing single-slot producers can keep winning the CAS, so a large frame + // has no progress bound under sustained contention. The cooperative drain keeps retrying + // (no deadlock), but latency is unbounded; revisit with a reservation scheme if it shows + // up in send stats. + fn try_send_multi_slot( + &self, + header: &DsmMpscRingHeader, + bytes: &[u8], + n_slots: u32, + payload_cap: usize, + ) -> Result<(), SendError> { + loop { + let tail = header.tail.load(Ordering::Acquire); + // Verify every target slot is currently in its expected empty round. + // Slot at position (tail + i) % ring_size in round-of-(tail+i) has empty + // marker == (tail + i). If ANY is not empty, the ring is too contended / + // not drained enough for a run of this length. + let mut all_empty = true; + for i in 0..n_slots { + let t = tail.wrapping_add(i as u64); + let slot_idx = (t % header.ring_size as u64) as u32; + let slot = unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let seq = unsafe { (*slot).seq.load(Ordering::Acquire) }; + match seq.cmp(&t) { + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Less => { + // Slot at offset i hasn't been reclaimed by the consumer for + // round-of-t. Run not available right now. + return Err(SendError::Full); + } + std::cmp::Ordering::Greater => { + // Another producer already claimed (T + i); our view of `tail` + // is stale, retry. + all_empty = false; + break; + } + } + } + if !all_empty { + continue; + } + // All N target slots are empty in our round. Atomically claim the run. + match header.tail.compare_exchange_weak( + tail, + tail.wrapping_add(n_slots as u64), + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + // We own slots tail..tail+n_slots. Write each fragment and + // publish its seq in ascending order so the receiver can drain + // them as soon as the prefix is ready. + let mut offset = 0usize; + for i in 0..n_slots { + let t = tail.wrapping_add(i as u64); + let slot_idx = (t % header.ring_size as u64) as u32; + let slot = + unsafe { slot_ptr(self.ring.as_ptr(), slot_idx, header.slot_capacity) }; + let remaining = bytes.len() - offset; + let chunk_len = remaining.min(payload_cap); + let kind = if i == 0 { + FragmentKind::First + } else if i + 1 == n_slots { + FragmentKind::Last + } else { + FragmentKind::Continue + }; + let flags_word = if kind == FragmentKind::First { + pack_flags(kind, n_slots) + } else { + pack_flags(kind, 0) + }; + unsafe { + (*slot).len.store(chunk_len as u32, Ordering::Relaxed); + (*slot).flags.store(flags_word, Ordering::Relaxed); + let data = slot_data_ptr(slot); + std::ptr::copy_nonoverlapping( + bytes.as_ptr().add(offset), + data, + chunk_len, + ); + // Publish: ready in round-of-t for slot at position t is t + 1. + (*slot).seq.store(t.wrapping_add(1), Ordering::Release); + } + offset += chunk_len; + } + debug_assert_eq!(offset, bytes.len(), "multi-slot send copied wrong length"); + // Wake the consumer once for the whole run; the drain pass that + // wakes for the First slot will keep going through all our + // already-published Continue/Last fragments without sleeping. + self.wake_receiver(); + return Ok(()); + } + Err(_) => continue, + } + } + } +} + +impl Drop for DsmMpscSender { + fn drop(&mut self) { + if !self.alive.load(Ordering::Acquire) { + // Segment detached: the `sender_count.fetch_sub` below would write into freed DSM. + return; + } + if !self.counts_as_data { + // Control senders never joined `sender_count`, so there's nothing to release and no + // detach to trigger. + return; + } + let header = unsafe { self.ring.as_ref() }; + // AcqRel: decrement is observed by other producers (they don't care, but the + // Release pairs with the receiver's Acquire load on `detached` below). + let prev = header.sender_count.fetch_sub(1, Ordering::AcqRel); + if prev == 1 { + // We were the last sender. Tell the consumer. + header.detached.store(true, Ordering::Release); + self.wake_receiver(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering as O}; + + /// No-op wakeup for the busy-poll tests, which spin on `try_recv` and never rely on being + /// woken. + struct NoopWakeup; + impl Wakeup for NoopWakeup { + fn wake(&self, _token: u64) {} + } + fn noop_wakeup() -> Arc { + Arc::new(NoopWakeup) + } + + /// In-process wakeup: unparks the consumer thread registered under a token. Proves the + /// injected extension point carries a real cross-thread notification with no PG / `SetLatch`. + #[derive(Default)] + struct ThreadWakeup { + threads: std::sync::Mutex>, + wakes: AtomicUsize, + } + impl ThreadWakeup { + fn register(&self, token: u64, thread: std::thread::Thread) { + self.threads.lock().unwrap().insert(token, thread); + } + } + impl Wakeup for ThreadWakeup { + fn wake(&self, token: u64) { + self.wakes.fetch_add(1, O::Relaxed); + if let Some(t) = self.threads.lock().unwrap().get(&token) { + t.unpark(); + } + } + } + + /// Send + Copy wrapper so tests can pass the ring pointer into spawned threads. The + /// real production handles (DsmMpscSender / Receiver) already impl Send; this exists + /// only because constructing them is unsafe and we want the test to do it inside the + /// thread so each thread has its own handle. + #[derive(Clone, Copy)] + struct SharedRing(NonNull); + unsafe impl Send for SharedRing {} + + /// Owning aligned region for a ring. The default `Vec` allocator can't + /// promise the alignment the `create_at` write wants, so allocate via the global + /// allocator with an explicit `Layout` and free in `Drop`. Production uses PG + /// `dsm_segment` (page-aligned), so this dance is test-only. + struct AlignedRegion { + ptr: *mut u8, + layout: std::alloc::Layout, + } + impl AlignedRegion { + fn new(bytes: usize) -> Self { + let align = std::mem::align_of::(); + let layout = std::alloc::Layout::from_size_align(bytes, align).expect("invalid layout"); + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null(), "allocator returned null"); + Self { ptr, layout } + } + fn as_mut_ptr(&self) -> *mut u8 { + self.ptr + } + } + impl Drop for AlignedRegion { + fn drop(&mut self) { + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } + } + + /// Allocate a heap region big enough for a ring with `ring_size` slots of + /// `slot_capacity` bytes each, initialize it, and return `(owner, receiver, + /// sender_template)`. The owner is returned first so callers bind it first. + /// Rust drops locals in reverse declaration order, so `_region` bound first drops + /// last, after the handles whose `Drop` impls touch the region's memory. Reverse + /// that order and `_region` frees the bytes before `tx_template`'s + /// `sender_count.fetch_sub` runs, which is undefined behavior and surfaces as a + /// stochastic crash at process teardown. + fn make_ring( + ring_size: u32, + slot_capacity: u32, + ) -> (AlignedRegion, DsmMpscReceiver, DsmMpscSender) { + let bytes = DsmMpscRingHeader::region_bytes(ring_size, slot_capacity); + let region = AlignedRegion::new(bytes); + let header_ptr = unsafe { create_at(region.as_mut_ptr(), ring_size, slot_capacity) }; + let nn = NonNull::new(header_ptr).expect("create_at returned null"); + // Unsafe: we hand ownership of the same pointer to two handles. Safe because the + // ring is the synchronization point; the handles are stateless wrappers. + let alive = Arc::new(AtomicBool::new(true)); + let receiver = unsafe { DsmMpscReceiver::new(nn, Arc::clone(&alive)) }; + let sender = unsafe { DsmMpscSender::new(nn, noop_wakeup(), alive) }; + (region, receiver, sender) + } + + #[test] + fn control_sender_does_not_gate_detach() { + let bytes = DsmMpscRingHeader::region_bytes(4, 64); + let region = AlignedRegion::new(bytes); + let nn = NonNull::new(unsafe { create_at(region.as_mut_ptr(), 4, 64) }).unwrap(); + let alive = Arc::new(AtomicBool::new(true)); + let rx = unsafe { DsmMpscReceiver::new(nn, Arc::clone(&alive)) }; + let data = unsafe { DsmMpscSender::new(nn, noop_wakeup(), alive) }; + let control = data.to_control(); + + let mut buf = Vec::new(); + // The control sender publishes without joining `sender_count`. + control.try_send(&[1, 2, 3]).unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(&buf[..], &[1, 2, 3]); + // The data sender still holds the ring open. + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + + // Dropping the only data sender detaches the ring even though the control sender is held, + // so a downstream cancel never masks a producer-gone signal. + drop(data); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Detached); + + // The control sender still reaches the now-detached inbox: a `Cancel` has to land on a + // producer whose own upstream already finished. + control.try_send(&[9]).unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(&buf[..], &[9]); + drop(control); + } + + #[test] + fn spsc_round_trip_under_capacity() { + let (_region, rx, tx) = make_ring(4, 64); + for i in 0..3u8 { + tx.try_send(&[i, i + 1, i + 2]).unwrap(); + } + let mut buf = Vec::new(); + for i in 0..3u8 { + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(&buf[..], &[i, i + 1, i + 2]); + } + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + } + + #[test] + fn fills_then_full_then_drains() { + let (_region, rx, tx) = make_ring(4, 64); + for i in 0..4u32 { + tx.try_send(&i.to_le_bytes()).unwrap(); + } + // Fifth send must fail Full. + assert_eq!(tx.try_send(&999u32.to_le_bytes()), Err(SendError::Full)); + // Drain one, send one more, drain rest. + let mut buf = Vec::new(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, 0u32.to_le_bytes()); + tx.try_send(&100u32.to_le_bytes()).unwrap(); + for expected in [1u32, 2, 3, 100] { + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, expected.to_le_bytes()); + } + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + } + + #[test] + fn message_too_large_is_rejected() { + // `MessageTooLarge` only fires when the frame would need more than `ring_size` + // slots. Per-slot oversize splits across consecutive slots instead. + let (_region, _rx, tx) = make_ring(2, 32); + let payload_cap = 32 - SLOT_HEADER_BYTES; + // payload_cap + 1 byte: now spans 2 slots, fits in ring_size=2. + let two_slot = vec![0u8; payload_cap + 1]; + tx.try_send(&two_slot).unwrap(); + // ring_size * payload_cap + 1 byte: needs 3 slots, ring only has 2. + let oversize = vec![0u8; 2 * payload_cap + 1]; + assert_eq!(tx.try_send(&oversize), Err(SendError::MessageTooLarge)); + // Exactly at the ring-wide cap is fine. (We just sent a 2-slot frame above, + // so we need to drain it first to free the slots; the receiver only exists + // in this test for that.) + } + + /// Multi-producer concurrent send: K threads each push M unique messages; consumer + /// receives K*M messages and verifies every (producer, sequence_in_producer) pair + /// appears exactly once. + #[test] + fn mpsc_no_lost_messages_under_contention() { + const K_PRODUCERS: usize = 8; + const M_PER_PRODUCER: u32 = 2000; + let (_region, rx, tx_template) = make_ring(64, 32); + // Wrap region pointer in something we can share across threads. The handles + // themselves are Send, so we clone via NonNull copy. + let ring_nn = SharedRing(tx_template.ring); + let consumed = Arc::new(AtomicUsize::new(0)); + let mut handles = Vec::with_capacity(K_PRODUCERS); + for producer_id in 0..K_PRODUCERS { + // Construct the sender on this thread so the closure captures DsmMpscSender + // (Send via unsafe impl) rather than the inner NonNull (Rust 2021 disjoint + // capture would otherwise project to the NonNull field and fail Send). + let tx = unsafe { + DsmMpscSender::new(ring_nn.0, noop_wakeup(), Arc::clone(&tx_template.alive)) + }; + let h = std::thread::spawn(move || { + let mut sent = 0u32; + while sent < M_PER_PRODUCER { + let mut payload = [0u8; 8]; + payload[0..4].copy_from_slice(&(producer_id as u32).to_le_bytes()); + payload[4..8].copy_from_slice(&sent.to_le_bytes()); + match tx.try_send(&payload) { + Ok(_) => sent += 1, + Err(SendError::Full) => std::thread::yield_now(), + Err(e) => panic!("unexpected send error: {e:?}"), + } + } + }); + handles.push(h); + } + // Drain on this thread until every producer's M messages have shown up. + let mut seen: Vec> = (0..K_PRODUCERS) + .map(|_| vec![false; M_PER_PRODUCER as usize]) + .collect(); + let mut buf = Vec::new(); + let target = K_PRODUCERS * M_PER_PRODUCER as usize; + while consumed.load(O::Relaxed) < target { + match rx.try_recv(&mut buf) { + RecvOutcome::Bytes => { + assert_eq!(buf.len(), 8); + let producer_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()) as usize; + let sent_idx = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + assert!(producer_id < K_PRODUCERS, "bad producer id {producer_id}"); + assert!( + sent_idx < M_PER_PRODUCER as usize, + "bad sent idx {sent_idx}" + ); + let already = std::mem::replace(&mut seen[producer_id][sent_idx], true); + assert!(!already, "duplicate ({producer_id}, {sent_idx})"); + consumed.fetch_add(1, O::Relaxed); + } + RecvOutcome::Empty => std::thread::yield_now(), + RecvOutcome::Detached => panic!("unexpected detach mid-drain"), + } + } + for h in handles { + h.join().unwrap(); + } + for (p, row) in seen.iter().enumerate() { + assert!( + row.iter().all(|&b| b), + "producer {p} has a missed message: row = {row:?}" + ); + } + } + + /// Per-producer in-order property: a single producer's messages observed by the + /// consumer must arrive in the order the producer sent them. (Cross-producer + /// ordering is not guaranteed.) + #[test] + fn mpsc_preserves_per_producer_order() { + const K_PRODUCERS: usize = 4; + const M_PER_PRODUCER: u32 = 500; + let (_region, rx, tx_template) = make_ring(32, 32); + let ring_nn = SharedRing(tx_template.ring); + let mut handles = Vec::with_capacity(K_PRODUCERS); + for producer_id in 0..K_PRODUCERS { + let tx = unsafe { + DsmMpscSender::new(ring_nn.0, noop_wakeup(), Arc::clone(&tx_template.alive)) + }; + handles.push(std::thread::spawn(move || { + let mut sent = 0u32; + while sent < M_PER_PRODUCER { + let mut payload = [0u8; 8]; + payload[0..4].copy_from_slice(&(producer_id as u32).to_le_bytes()); + payload[4..8].copy_from_slice(&sent.to_le_bytes()); + if tx.try_send(&payload).is_ok() { + sent += 1; + } else { + std::thread::yield_now(); + } + } + })); + } + let mut last_seq: Vec = vec![-1; K_PRODUCERS]; + let mut buf = Vec::new(); + let mut total = 0usize; + let target = K_PRODUCERS * M_PER_PRODUCER as usize; + while total < target { + match rx.try_recv(&mut buf) { + RecvOutcome::Bytes => { + let producer_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()) as usize; + let sent_idx = i64::from(u32::from_le_bytes(buf[4..8].try_into().unwrap())); + assert_eq!( + sent_idx, + last_seq[producer_id] + 1, + "out-of-order from producer {producer_id}: got {sent_idx}, expected {}", + last_seq[producer_id] + 1 + ); + last_seq[producer_id] = sent_idx; + total += 1; + } + RecvOutcome::Empty => std::thread::yield_now(), + RecvOutcome::Detached => panic!("unexpected detach"), + } + } + for h in handles { + h.join().unwrap(); + } + } + + /// Cache-line layout regression: `head` must land on its own cache line + /// (offset == CACHE_LINE so the preceding header fields can't false-share), + /// `tail` separated from `head` by CACHE_LINE, and the first slot not sharing a + /// line with `tail`. Catches accidental padding removal or field reorder that + /// would re-introduce the false-sharing perf cliff the padding exists to avoid. + #[test] + fn header_layout_is_cache_friendly() { + use std::mem::offset_of; + let head_off = offset_of!(DsmMpscRingHeader, head); + let tail_off = offset_of!(DsmMpscRingHeader, tail); + let header_size = std::mem::size_of::(); + // head at exactly CACHE_LINE offset means the preceding 64 bytes (header + // fields + pad) live entirely in cache line 0, head + its pad live in + // cache line 1, etc. Anything other than equality means the padding + // math drifted; fix it rather than relaxing the assertion. + assert_eq!( + head_off, CACHE_LINE, + "head must land at offset {CACHE_LINE} (its own cache line); got {head_off}" + ); + assert!( + (tail_off - head_off) >= CACHE_LINE, + "head and tail must be on separate cache lines: head_off={head_off}, tail_off={tail_off}, CACHE_LINE={CACHE_LINE}" + ); + assert!( + (header_size - tail_off) >= CACHE_LINE, + "first slot must start on its own cache line: header_size={header_size}, tail_off={tail_off}, CACHE_LINE={CACHE_LINE}" + ); + // Header's natural alignment is determined by its largest field (AtomicU64, + // 8 bytes). Cache-line padding between hot fields still keeps them on separate + // absolute lines regardless of the struct's starting alignment, as long as the + // inter-field distance is >= CACHE_LINE. + assert_eq!(std::mem::align_of::(), 8); + } + + /// Magic + version validation: an attach against a region with the wrong magic or + /// version returns None rather than handing back a NonNull with garbage state. + /// Mirrors `MppDsmHeader::validate`'s discipline. + #[test] + fn attach_at_rejects_wrong_magic_and_version() { + let bytes = DsmMpscRingHeader::region_bytes(4, 64); + let region = AlignedRegion::new(bytes); + let header_ptr = unsafe { create_at(region.as_mut_ptr(), 4, 64) }; + // Sanity: correct attach succeeds. + assert!(unsafe { attach_at(region.as_mut_ptr(), 4, 64) }.is_some()); + // Corrupt magic: rejected. + unsafe { (*header_ptr).magic = 0xDEADBEEF }; + assert!(unsafe { attach_at(region.as_mut_ptr(), 4, 64) }.is_none()); + // Restore magic, corrupt version. + unsafe { (*header_ptr).magic = MPSC_RING_MAGIC }; + unsafe { (*header_ptr).version = MPSC_RING_VERSION.wrapping_add(1) }; + assert!(unsafe { attach_at(region.as_mut_ptr(), 4, 64) }.is_none()); + // Restore version, mismatched ring_size. + unsafe { (*header_ptr).version = MPSC_RING_VERSION }; + assert!(unsafe { attach_at(region.as_mut_ptr(), 8, 64) }.is_none()); + } + + /// Stress test at the production-worst contention level (K=24 producers, matching + /// the largest mesh-row size the transport drives in practice). Smoke that the + /// primitive doesn't wedge or lose messages under heavy CAS contention; doesn't + /// measure perf. + #[test] + fn mpsc_stress_at_production_worst_case() { + const K_PRODUCERS: usize = 24; + const M_PER_PRODUCER: u32 = 500; + let (_region, rx, tx_template) = make_ring(64, 32); + let ring_nn = SharedRing(tx_template.ring); + let mut handles = Vec::with_capacity(K_PRODUCERS); + for producer_id in 0..K_PRODUCERS { + let tx = unsafe { + DsmMpscSender::new(ring_nn.0, noop_wakeup(), Arc::clone(&tx_template.alive)) + }; + handles.push(std::thread::spawn(move || { + let mut payload = [0u8; 8]; + payload[0..4].copy_from_slice(&(producer_id as u32).to_le_bytes()); + let mut sent = 0u32; + while sent < M_PER_PRODUCER { + payload[4..8].copy_from_slice(&sent.to_le_bytes()); + match tx.try_send(&payload) { + Ok(_) => sent += 1, + Err(SendError::Full) => std::thread::yield_now(), + Err(e) => panic!("unexpected: {e:?}"), + } + } + })); + } + let mut buf = Vec::new(); + let target = K_PRODUCERS * M_PER_PRODUCER as usize; + let mut total = 0usize; + while total < target { + match rx.try_recv(&mut buf) { + RecvOutcome::Bytes => total += 1, + RecvOutcome::Empty => std::thread::yield_now(), + RecvOutcome::Detached => panic!("unexpected detach"), + } + } + for h in handles { + h.join().unwrap(); + } + // Multi-round invariant: after draining 24 * 500 = 12000 messages on a 64-slot + // ring, every slot's seq must have advanced into a future round. Walk slot[0]: + // we expect seq = head + ring_size = 12000 + 64 - whatever round 0 it's in. + // Loose check: every slot's seq is bounded below by ring_size (round >= 1). + let header = unsafe { ring_nn.0.as_ref() }; + for i in 0..header.ring_size { + let slot = unsafe { slot_ptr(ring_nn.0.as_ptr(), i, header.slot_capacity) }; + let seq = unsafe { (*slot).seq.load(O::Acquire) }; + assert!( + seq >= header.ring_size as u64, + "slot[{i}].seq={seq} < ring_size; slot was never reused" + ); + } + } + + // ----- multi-slot fragmentation tests ----- + + /// Two-slot frame round-trip: send a payload that's exactly `2*payload_cap` + /// bytes long with distinct first-half and second-half markers, then verify the + /// receiver reassembles them in order with no truncation or duplication. + #[test] + fn multi_slot_two_fragment_round_trip() { + let slot_cap = 64; + let payload_cap = slot_cap - SLOT_HEADER_BYTES; + let (_region, rx, tx) = make_ring(4, slot_cap as u32); + let mut frame = vec![0u8; 2 * payload_cap]; + for (i, b) in frame.iter_mut().enumerate() { + *b = (i & 0xFF) as u8; + } + tx.try_send(&frame).unwrap(); + let mut buf = Vec::new(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf.len(), frame.len()); + assert_eq!(buf, frame); + // Ring fully drained. + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + } + + /// Frame at the multi-slot ceiling: needs exactly `ring_size` slots, succeeds; + /// one byte more, rejected with MessageTooLarge. + #[test] + fn multi_slot_max_size_succeeds_one_more_rejected() { + let slot_cap = 64; + let payload_cap = slot_cap - SLOT_HEADER_BYTES; + let ring_size = 4; + let (_region, rx, tx) = make_ring(ring_size, slot_cap as u32); + let max = vec![0xABu8; ring_size as usize * payload_cap]; + tx.try_send(&max).unwrap(); + let mut buf = Vec::new(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, max); + let too_big = vec![0u8; ring_size as usize * payload_cap + 1]; + assert_eq!(tx.try_send(&too_big), Err(SendError::MessageTooLarge)); + } + + /// Multi-slot followed by single-slot from the same producer: each frame must + /// come out of the receiver as a separate, intact byte sequence; fragments + /// don't leak into the following frame. + #[test] + fn multi_slot_then_single_slot_preserves_boundaries() { + let slot_cap = 64; + let payload_cap = slot_cap - SLOT_HEADER_BYTES; + let (_region, rx, tx) = make_ring(8, slot_cap as u32); + let big = vec![0xAAu8; 3 * payload_cap]; + tx.try_send(&big).unwrap(); + let small = b"hi".to_vec(); + tx.try_send(&small).unwrap(); + let mut buf = Vec::new(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, big); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, small); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + } + + /// Multi-slot frame wrapping the ring boundary. With ring_size=4, tail starts at 0: + /// send a single-slot first to advance tail to 1, drain it, then send a 3-slot + /// frame at tail=1 occupying slots 1, 2, 3, drain the small frame to advance head, + /// then send another single-slot at tail=4 (slot 0 in round 1) and a 3-slot at + /// tail=5 (slots 1, 2, 3 in round 1). Verifies wraparound math holds. + #[test] + fn multi_slot_wraparound() { + let slot_cap = 64; + let payload_cap = slot_cap - SLOT_HEADER_BYTES; + let (_region, rx, tx) = make_ring(4, slot_cap as u32); + let mut buf = Vec::new(); + // Push a small frame so the next multi-slot reservation starts mid-ring. + tx.try_send(b"warm").unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, b"warm"); + // 3-slot frame at tail=1, wrapping at slot 3 -> slot 0 won't happen here + // (slots 1, 2, 3 are still in round 0). Drain it. + let big = vec![0x11u8; 3 * payload_cap]; + tx.try_send(&big).unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, big); + // Now tail=4, head=4. Send another small frame at slot 0 (round 1, seq=4). + tx.try_send(b"two").unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, b"two"); + // And a 3-slot at tail=5 occupying slots 1, 2, 3 in round 1 (seqs 5, 6, 7). + let big2 = vec![0x22u8; 3 * payload_cap]; + tx.try_send(&big2).unwrap(); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Bytes); + assert_eq!(buf, big2); + assert_eq!(rx.try_recv(&mut buf), RecvOutcome::Empty); + } + + /// Stress: multiple producers each push a mix of single-slot and multi-slot + /// frames; consumer verifies every frame's contents and per-producer ordering. + /// Catches interleave bugs (a multi-slot run getting mixed with another + /// producer's fragments) and missing wake-ups under high contention. + #[test] + fn multi_slot_stress_mixed_with_singles() { + const K_PRODUCERS: usize = 6; + const M_PER_PRODUCER: u32 = 300; + let slot_cap = 128; + let payload_cap = slot_cap - SLOT_HEADER_BYTES; + let (_region, rx, tx_template) = make_ring(16, slot_cap as u32); + let ring_nn = SharedRing(tx_template.ring); + let mut handles = Vec::with_capacity(K_PRODUCERS); + for producer_id in 0..K_PRODUCERS { + let tx = unsafe { + DsmMpscSender::new(ring_nn.0, noop_wakeup(), Arc::clone(&tx_template.alive)) + }; + handles.push(std::thread::spawn(move || { + let mut sent = 0u32; + while sent < M_PER_PRODUCER { + // Alternate single, 2-slot, 4-slot, single, ... so the consumer + // sees every fragment-kind combination. + let n_payload = match sent % 3 { + 0 => 8, // single-slot + 1 => 2 * payload_cap, // 2-slot + _ => 4 * payload_cap, // 4-slot + }; + let mut payload = vec![0u8; n_payload + 8]; + payload[0..4].copy_from_slice(&(producer_id as u32).to_le_bytes()); + payload[4..8].copy_from_slice(&sent.to_le_bytes()); + for (i, b) in payload[8..].iter_mut().enumerate() { + *b = ((producer_id ^ sent as usize ^ i) & 0xFF) as u8; + } + match tx.try_send(&payload) { + Ok(_) => sent += 1, + Err(SendError::Full) => std::thread::yield_now(), + Err(e) => panic!("unexpected send error: {e:?}"), + } + } + })); + } + let mut last_seq: Vec = vec![-1; K_PRODUCERS]; + let mut buf = Vec::new(); + let mut total = 0usize; + let target = K_PRODUCERS * M_PER_PRODUCER as usize; + while total < target { + match rx.try_recv(&mut buf) { + RecvOutcome::Bytes => { + assert!(buf.len() >= 8, "frame too short: {}", buf.len()); + let producer_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()) as usize; + let sent_idx = i64::from(u32::from_le_bytes(buf[4..8].try_into().unwrap())); + assert!(producer_id < K_PRODUCERS, "bad producer id"); + assert_eq!( + sent_idx, + last_seq[producer_id] + 1, + "out-of-order frame from producer {producer_id}" + ); + // Verify payload bytes (catches fragment interleave / partial reads). + for (i, b) in buf[8..].iter().enumerate() { + let expected = ((producer_id ^ sent_idx as usize ^ i) & 0xFF) as u8; + assert_eq!(*b, expected, "payload mismatch at byte {i}"); + } + last_seq[producer_id] = sent_idx; + total += 1; + } + RecvOutcome::Empty => std::thread::yield_now(), + RecvOutcome::Detached => panic!("unexpected detach"), + } + } + for h in handles { + h.join().unwrap(); + } + } + + /// The injected `Wakeup` carries a real cross-thread notification with no PG: a consumer that + /// parks until woken is unparked by the producer's publish, and the producer actually invoked + /// the wakeup (asserted via the counter, so a silently-broken extension point fails rather than relying + /// on park timing). + #[test] + fn injected_wakeup_unparks_blocked_consumer() { + let region = AlignedRegion::new(DsmMpscRingHeader::region_bytes(8, 256)); + let nn = NonNull::new(unsafe { create_at(region.as_mut_ptr(), 8, 256) }) + .expect("create_at returned null"); + let ring = SharedRing(nn); + let wakeup = Arc::new(ThreadWakeup::default()); + let alive = Arc::new(AtomicBool::new(true)); + const TOKEN: u64 = 7; + + let receiver = unsafe { DsmMpscReceiver::new(nn, Arc::clone(&alive)) }; + // Register this thread as the wake target before publishing the token, so a producer that + // races ahead still finds us. + wakeup.register(TOKEN, std::thread::current()); + receiver.set_receiver(TOKEN); + + let producer = { + let wakeup = Arc::clone(&wakeup) as Arc; + let alive = Arc::clone(&alive); + std::thread::spawn(move || { + // Rebind the whole `Send` wrapper so the closure captures it (not the bare + // `NonNull` field, which edition-2024 disjoint capture would otherwise grab). + let ring = ring; + let tx = unsafe { DsmMpscSender::new(ring.0, wakeup, alive) }; + std::thread::sleep(std::time::Duration::from_millis(20)); + assert_eq!(tx.try_send(b"hello"), Ok(())); + }) + }; + + // park_timeout (not park) so a broken extension point fails via the counter assertion below instead + // of hanging the suite. park/unpark holds a token, so an unpark that beats our park still + // wakes the next park (no lost-wakeup). + let mut out = Vec::new(); + loop { + match receiver.try_recv(&mut out) { + RecvOutcome::Bytes => break, + RecvOutcome::Empty => std::thread::park_timeout(std::time::Duration::from_secs(5)), + RecvOutcome::Detached => panic!("detached before receiving the frame"), + } + } + producer.join().unwrap(); + assert_eq!(out, b"hello"); + assert!( + wakeup.wakes.load(O::Relaxed) >= 1, + "producer never invoked the injected wakeup" + ); + } +} diff --git a/src/shm/runtime.rs b/src/shm/runtime.rs new file mode 100644 index 000000000..6b6169051 --- /dev/null +++ b/src/shm/runtime.rs @@ -0,0 +1,551 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runtime glue between the leader's DataFusion execution and the DSM MPSC mesh. +//! +//! [`MppMesh`] is the runtime handle the leader builds at DSM-init time. It holds the +//! single [`super::transport::DrainHandle`] (the +//! `inbound_receiver`) that consolidates this proc's DSM inbox and self-loop, and gets +//! installed on the leader's `SessionConfig` extensions before plan execution. +//! +//! [`ShmChannelResolver`] implements [`ChannelResolver`], consulted by +//! `NetworkShuffleExec` / `NetworkCoalesceExec` / `NetworkBroadcastExec` at execute time. It hands +//! out a [`ShmWorkerChannel`] whose `execute_task(task_key, partition_range)` yields one stream per +//! consumer partition from the shared `inbound_receiver`. +//! +//! [`InProcessWorkerResolver`] hands the planner `n_workers` placeholder URLs. The transport +//! routes by task index, not URL, so the URLs are never dialed; the resolver exists only because +//! the planner sizes stages from the URL count. It replaces the placeholder URL the fork used to +//! substitute internally under the old `in_process_mode` flag. + +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::{DataFusionError, Result, internal_err}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder}; +use futures::stream::{self, BoxStream, StreamExt}; +use http::HeaderMap; +use url::Url; + +use crate::common::serialize_uuid; +use crate::proto as pb; +use crate::work_unit_feed::RemoteWorkUnitFeedTxs; +use crate::{ + ChannelResolver, CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, + GetWorkerInfoResponse, WorkerChannel, WorkerResolver, WorkerToCoordinatorMsg, +}; + +use super::AliveFlag; +use super::transport::{ + CooperativeDrainSet, DrainHandle, DrainItem, Interrupt, MppFrameHeader, MppSender, + SendBatchStats, SetPlanFrame, +}; + +/// A proc's outbound senders to each peer inbox, shared between the mesh (for `Cancel` frames) and +/// the embedder that owns their lifetime. Indexed by destination `proc_idx`; this proc's own slot +/// is `None`. +pub type PeerSenders = Arc>>>; + +/// `task_idx → proc_idx` round-robin over the worker procs. The leader is `proc_idx = 0` +/// (consumer-only), workers are `1..n_procs` (each hosts producer fragments). +/// +/// A stage's task count is set by the DF-D task estimator chain, not by the worker proc count. +/// The transport does NOT support more tasks than producer procs: channels are keyed +/// `(sender_proc, stage, partition)`, so two tasks wrapped onto one proc would interleave on one +/// channel and the first EOF would truncate the other task's output. `ShmWorkerChannel::execute_task` +/// rejects that shape; the modulo here only keeps the function total. +#[inline] +pub fn proc_for_task(n_workers: u32, task_idx: u32) -> u32 { + 1 + (task_idx % n_workers.max(1)) +} + +/// Runtime handle the customscan populates at DSM-init time. +/// +/// Each process owns one MPSC inbox in DSM that receives frames from every peer. +/// `inbound_receiver` consolidates that inbox plus the in-proc self-loop channel (for +/// producer-and-consumer-on-same-worker fragments) into a single `DrainHandle`. Frames +/// carry `(sender_proc, stage_id, partition)` in their header so the routing registry +/// inside the handle delivers each frame to the matching consumer. +/// +/// [`MppFrameHeader`]: super::transport::MppFrameHeader +pub struct MppMesh { + /// This process's `proc_idx` (= 0 for the leader, `ParallelWorkerNumber + 1` for workers). + /// Frames addressed to this proc arrive on this proc's own inbox. + pub this_proc: u32, + /// Total proc count. Bounds the producer/consumer proc lookups in + /// [`ShmWorkerChannel::execute_task`]. + pub n_procs: u32, + /// Single cooperative inbound handle pulling every frame addressed to this proc. The + /// DSM MPSC inbox and an in-proc self-loop receiver both feed into this handle. Demux + /// to per-`(sender_proc, stage_id, partition)` channel buffers happens inside via + /// `DrainHandle::register_channel`. + pub(super) inbound_receiver: Arc, + /// Cancellation hook, injected by the embedder, checked at the transport's block points (the + /// cooperative send spin and the consumer pull loop). + interrupt: Arc, + /// This proc's outbound senders to each peer inbox, shared with the embedder that owns their + /// lifetime (it clears the `Vec` before the DSM unmaps). Used by [`Self::cancel_stream`] to ship + /// a `Cancel` frame to a producing proc when this proc abandons a stream. `None` until the + /// embedder installs it. + cancel_senders: Mutex>, + alive: AliveFlag, +} + +impl MppMesh { + /// Build a fresh mesh. + pub fn new( + this_proc: u32, + n_procs: u32, + inbound_receiver: Arc, + interrupt: Arc, + alive: AliveFlag, + ) -> Self { + Self { + this_proc, + n_procs, + inbound_receiver, + interrupt, + cancel_senders: Mutex::new(None), + alive, + } + } + + /// Mark this proc's DSM mesh detached so every ring handle's operations become no-ops. + /// The embedder calls this from its dsm-detach callback, while the segment is still mapped, + /// so handles dropped afterward (e.g. by a memory-context reset) never touch freed memory. + pub fn mark_detached(&self) { + self.alive.store(false, Ordering::Release); + } + + /// The raw liveness flag shared with every ring handle, for embedders that register a C + /// dsm-detach callback against it directly. + pub fn detached_flag(&self) -> AliveFlag { + Arc::clone(&self.alive) + } + + /// Share this proc's outbound senders so [`Self::cancel_stream`] can reach the producers. The + /// embedder keeps owning their lifetime: it passes a clone of the same `Arc` it releases before + /// the DSM unmaps, so the mesh never extends the senders past teardown. + pub fn set_cancel_senders(&self, senders: PeerSenders) { + *self.cancel_senders.lock().unwrap() = Some(senders); + } + + /// Route a leader-built `SetPlan` frame to the worker proc that owns `task_number`, over the + /// leader's outbound senders (installed via [`Self::set_cancel_senders`]). This is the unified + /// dispatch path: the coordinator ships each stage's plan through here, so the embedder no + /// longer routes it out of band. Errors if the senders aren't installed or the destination + /// slot is empty; the caller treats a failure as best-effort (the worker starves on its + /// `SetPlan` wait, which the dispatch deadlock detector surfaces). + pub async fn send_set_plan( + self: &Arc, + stage_id: u32, + task_number: u32, + frame: SetPlanFrame, + ) -> Result<()> { + let dest_proc = proc_for_task(self.n_workers(), task_number); + let sender = { + let guard = self.cancel_senders.lock().unwrap(); + let Some(senders) = guard.as_ref() else { + return internal_err!( + "shm mesh: control senders not installed; cannot route SetPlan" + ); + }; + let senders = senders.lock().unwrap(); + let Some(Some(base)) = senders.get(dest_proc as usize) else { + return internal_err!("shm mesh: no control sender for proc {dest_proc}"); + }; + base.clone_with_header(MppFrameHeader::set_plan(stage_id, task_number, 0)) + .with_cooperative_drain(Arc::clone(self) as Arc) + }; + let mut stats = SendBatchStats::default(); + sender.send_set_plan_traced(&frame, &mut stats).await + } + + /// Tell the producer on `producer_proc` to stop the `(stage_id, partition)` stream: this proc's + /// consumer of it stopped reading before EOF. Ships one `Cancel` frame, leaving the rings + /// healthy for metrics and every other stream. A no-op when no senders are installed (the + /// embedder hasn't wired this proc, or teardown cleared them). + /// + /// Stream-level so any consumer can cancel its own input: every `(producer_proc, stage, + /// partition)` channel has a single consumer, so one stream's drop never cuts off a sibling's. + pub fn cancel_stream(&self, producer_proc: u32, stage_id: u32, partition: u32) { + let guard = self.cancel_senders.lock().unwrap(); + let Some(senders) = guard.as_ref() else { + return; + }; + let senders = senders.lock().unwrap(); + if let Some(Some(sender)) = senders.get(producer_proc as usize) { + sender.try_send_cancel(stage_id, partition); + } + } + + /// The single cooperative inbound handle that pulls frames from every peer (and the + /// self-loop) into per-`(sender_proc, stage_id, partition)` channel buffers. + pub(super) fn inbound_receiver(&self) -> &Arc { + &self.inbound_receiver + } + + /// Install the senders of one task's work-unit feed channels on this proc's drain, so + /// inbound `WorkUnit` frames for `(stage_id, task_number)` flow into them. Units that + /// arrived first are flushed; a `FeedEof` that already came through closes the channels + /// immediately. The drain only fills channels: something on this proc must keep draining + /// (a consumer pull loop, a producer send spin, or an explicit + /// [`CooperativeDrainSet::try_drain_pass`] pump) or a fragment blocked on its feed starves. + pub fn register_work_unit_senders( + &self, + stage_id: u32, + task_number: u32, + senders: RemoteWorkUnitFeedTxs, + ) { + self.inbound_receiver + .register_work_unit_senders(stage_id, task_number, senders); + } + + /// Take the plan delivered for `(stage_id, task_number)` as a `SetPlan` frame on this proc's + /// inbox, waiting for it if it has not arrived yet. Something on this proc must keep + /// draining (a pump, a consumer pull loop, or a cooperative send spin) or the wait starves. + pub async fn take_set_plan( + &self, + stage_id: u32, + task_number: u32, + ) -> Result { + self.inbound_receiver + .take_set_plan(stage_id, task_number) + .await + } + + /// Take the stream of `TaskMetrics` frames arriving on this proc's inbox: + /// `(stage_id, task_number, metrics)` per producer task that reported in. Meant for the + /// leader; the first caller gets it, later calls get `None`. + pub fn take_task_metrics_receiver( + &self, + ) -> Option> { + self.inbound_receiver.take_task_metrics_receiver() + } + + /// Number of worker procs (= `n_procs - 1`, since the leader is proc 0). Used as the + /// modulus in [`proc_for_task`]. + /// + /// The embedder must guarantee `n_procs >= 2` (one consumer-only leader plus at least one + /// producer worker) before constructing an `MppMesh`; the subtraction is otherwise unsound. + /// `compute_dsm_layout` enforces the same bound. Asserted in debug builds so a future misuse + /// fails loudly. + pub fn n_workers(&self) -> u32 { + debug_assert!( + self.n_procs >= 2, + "MppMesh::n_workers() called with n_procs={} (< 2); the embedder must gate on \ + n_procs >= 2", + self.n_procs + ); + self.n_procs - 1 + } + + /// Pull from the single inbound handle. Called from + /// [`super::transport::MppSender`]'s cooperative-send spin so a + /// producer stalled on a full outbound can pull inbound peer data inline. That's what + /// prevents the symmetric-send deadlock when every peer is simultaneously stalled waiting + /// for space. + pub(super) fn drain_all_inbound(&self) -> Result<(), DataFusionError> { + self.inbound_receiver.try_drain_pass() + } +} + +impl CooperativeDrainSet for MppMesh { + fn try_drain_pass(&self) -> Result<(), DataFusionError> { + self.drain_all_inbound() + } + + fn check_interrupt(&self) -> Result<(), DataFusionError> { + self.interrupt.check() + } + + fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { + self.inbound_receiver.stream_cancelled(stage_id, partition) + } +} + +/// Hands out [`ShmWorkerChannel`]s over the leader's [`MppMesh`]. The shm mesh has no URLs to dial, +/// so every `get_worker_client_for_url` resolves to the same mesh; the channel routes by the +/// `task_key` it is handed at `execute_task`, not by URL. +pub struct ShmChannelResolver { + mesh: Arc, +} + +impl ShmChannelResolver { + pub fn new(mesh: Arc) -> Self { + Self { mesh } + } +} + +#[async_trait] +impl ChannelResolver for ShmChannelResolver { + async fn get_worker_client_for_url( + &self, + _url: &Url, + ) -> Result, DataFusionError> { + Ok(Box::new(ShmWorkerChannel { + mesh: Arc::clone(&self.mesh), + })) + } +} + +/// A [`WorkerChannel`] over the mesh. `execute_task((stage, task), partition_range)` translates the +/// DF-D `(stage, task)` addressing into the proc-pair grid: `proc_for_task(n_workers, task_number)` +/// selects which `sender_proc` hosts the producer-side task, and each returned stream pulls that +/// proc's `(sender_proc, stage_id, partition)` slice from the inbound drain. +struct ShmWorkerChannel { + mesh: Arc, +} + +#[async_trait] +impl WorkerChannel for ShmWorkerChannel { + /// Route each `SetPlanRequest` the coordinator ships to the worker proc that owns its task, as a + /// `SetPlan` frame the worker picks up with `take_set_plan`. The coordinator's keep-alive tail + /// emits nothing after the request, so the loop then blocks until the stream closes, draining it + /// to exhaustion. Completes with an empty output stream: task metrics travel back over the mesh, + /// not here. + async fn coordinator_channel( + &mut self, + headers: HeaderMap, + mut c2w_stream: BoxStream<'static, CoordinatorToWorkerMsg>, + ) -> Result>> { + let mesh = Arc::clone(&self.mesh); + #[allow(clippy::disallowed_methods)] + tokio::spawn(async move { + while let Some(msg) = c2w_stream.next().await { + let CoordinatorToWorkerMsg::SetPlanRequest(req) = msg else { + continue; + }; + let stage_id = req.task_key.stage_id as u32; + let task_number = req.task_key.task_number as u32; + let set_plan = pb::SetPlanRequest { + task_key: Some(pb::TaskKey { + query_id: serialize_uuid(&req.task_key.query_id), + stage_id: req.task_key.stage_id as u64, + task_number: req.task_key.task_number as u64, + }), + task_count: req.task_count as u64, + plan_proto: req.plan_proto, + work_unit_feed_declarations: req + .work_unit_feed_declarations + .into_iter() + .map(|d| pb::set_plan_request::WorkUnitFeedDeclaration { + id: serialize_uuid(&d.id), + partitions: d.partitions as u64, + }) + .collect(), + target_worker_url: req.target_worker_url.to_string(), + query_start_time_ns: req.query_start_time_ns as u64, + }; + let frame = match SetPlanFrame::from_parts(set_plan, &headers) { + Ok(frame) => frame, + Err(e) => { + log::warn!("shm coordinator_channel: SetPlan frame build failed: {e}"); + continue; + } + }; + if let Err(e) = mesh.send_set_plan(stage_id, task_number, frame).await { + log::warn!("shm coordinator_channel: SetPlan route failed: {e}"); + } + } + }); + Ok(stream::empty().boxed()) + } + + async fn execute_task( + &mut self, + _headers: HeaderMap, + request: ExecuteTaskRequest, + metrics: ExecutionPlanMetricsSet, + _task_ctx: &Arc, + ) -> Result>>> { + MetricBuilder::new(&metrics) + .global_counter("mesh_connections_used") + .add(1); + let stage_id = u32::try_from(request.task_key.stage_id).map_err(|_| { + DataFusionError::Internal(format!( + "ShmWorkerChannel: stage_id={} > u32::MAX", + request.task_key.stage_id + )) + })?; + let task_number = u32::try_from(request.task_key.task_number).map_err(|_| { + DataFusionError::Internal(format!( + "ShmWorkerChannel: task_number={} > u32::MAX", + request.task_key.task_number + )) + })?; + let sender_proc = proc_for_task(self.mesh.n_workers(), task_number); + // More tasks than producer procs would fold two tasks onto one + // `(sender_proc, stage, partition)` channel: interleaved batches, and the first task's EOF + // truncates the second. With no input_stage here to count tasks, the equivalent guard is + // that the routed proc stays in range, as the old code also checked. + if sender_proc >= self.mesh.n_procs { + return Err(DataFusionError::Internal(format!( + "ShmWorkerChannel: sender_proc={sender_proc} >= n_procs={} \ + (stage_id={stage_id}, task_number={task_number})", + self.mesh.n_procs + ))); + } + // First line to grep when a query hangs on a channel nothing registered. + log::debug!( + "shm transport execute_task: this_proc={} stage_id={stage_id} \ + task_number={task_number} -> sender_proc={sender_proc}", + self.mesh.this_proc + ); + let mut streams = Vec::with_capacity( + request + .target_partition_end + .saturating_sub(request.target_partition_start), + ); + for partition in request.target_partition_start..request.target_partition_end { + let partition_u32 = u32::try_from(partition).map_err(|_| { + DataFusionError::Internal(format!( + "ShmWorkerChannel: partition={partition} > u32::MAX" + )) + })?; + streams.push(pull_partition_stream( + Arc::clone(&self.mesh), + sender_proc, + stage_id, + partition_u32, + )); + } + Ok(streams) + } + + async fn get_worker_info( + &mut self, + _request: GetWorkerInfoRequest, + ) -> Result { + Ok(GetWorkerInfoResponse { + version: String::new(), + }) + } +} + +/// Build the cooperative pull-loop stream for one `(producer_proc, stage_id, partition)` channel. +/// +/// The inbound drain runs inline on the consumer's thread. Each iteration checks for cancellation +/// (via the injected interrupt extension point), drains the receiver into the registry, pops one +/// batch to yield, then yields back to Tokio so sibling tasks (e.g. the leader's own producer +/// subplan) advance. The send side runs the same interrupt check inside `MppSender`'s retry spin. +fn pull_partition_stream( + mesh: Arc, + producer_proc: u32, + stage_id: u32, + partition: u32, +) -> BoxStream<'static, Result> { + // One drain per process, shared across all sender_procs. The channel-buffer registry keys by + // (sender_proc, stage_id, partition) so this consumer still sees only its named sender's slice + // even though the underlying inbox is shared with all peers. + let drain = Arc::clone(mesh.inbound_receiver()); + log::debug!( + "shm transport execute: register channel sender_proc={producer_proc} stage_id={stage_id} \ + partition={partition}" + ); + let buffer = drain.register_channel(producer_proc, stage_id, partition); + let stream = async_stream::stream! { + // Any consumer cancels its own input stream when it drops early. The mesh no-ops the send + // until the embedder wires this proc's outbound senders. + let mut cancel_guard = StreamCancelGuard { + mesh: Arc::clone(&mesh), + producer_proc, + stage_id, + partition, + armed: true, + }; + loop { + if let Err(e) = mesh.check_interrupt() { + yield Err(e); + return; + } + if let Err(e) = drain.try_drain_pass() { + yield Err(e); + return; + } + match buffer.try_pop() { + Some(DrainItem::Batch(batch)) => yield Ok(batch), + Some(DrainItem::Eof) => { + // Clean end: the producer already EOF'd, so there's nothing to cancel. + cancel_guard.armed = false; + break; + } + Some(DrainItem::Failed(msg)) => { + yield Err(DataFusionError::Execution(msg)); + return; + } + None => tokio::task::yield_now().await, + } + } + }; + Box::pin(stream) +} + +/// Placeholder worker resolver for the in-process MPP transport. +/// +/// The shm_mq transport routes by `target_task` (proc index), never by URL, so these URLs are +/// never dialed. The distributed planner still needs a resolver: it sizes stages and assigns +/// tasks from the URL count. `n_workers` placeholder URLs is exactly what the planner needs. This +/// replaces the placeholder URL the fork used to substitute internally under `in_process_mode`. +pub struct InProcessWorkerResolver { + n_workers: usize, +} + +impl InProcessWorkerResolver { + pub fn new(n_workers: usize) -> Self { + Self { n_workers } + } +} + +impl WorkerResolver for InProcessWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + (0..self.n_workers.max(1)) + .map(|i| { + Url::parse(&format!("inprocess://worker/{i}")).map_err(|e| { + DataFusionError::Internal(format!( + "InProcessWorkerResolver: invalid placeholder url: {e}" + )) + }) + }) + .collect() + } +} + +/// Cancels one `(stage_id, partition)` stream if its consumer drops before EOF, telling the +/// producer on `producer_proc` to stop. A consumer that stops pulling early (a top-N `LIMIT`, an +/// inner merge join exhausting a side, etc.) would otherwise leave that producer spinning on the +/// full inbox until the statement timeout. Disarmed on a clean EOF: there the producer already +/// finished, so there's nothing to cancel. +struct StreamCancelGuard { + mesh: Arc, + producer_proc: u32, + stage_id: u32, + partition: u32, + armed: bool, +} + +impl Drop for StreamCancelGuard { + fn drop(&mut self) { + if self.armed { + self.mesh + .cancel_stream(self.producer_proc, self.stage_id, self.partition); + } + } +} diff --git a/src/shm/self_hosted.rs b/src/shm/self_hosted.rs new file mode 100644 index 000000000..a14b00378 --- /dev/null +++ b/src/shm/self_hosted.rs @@ -0,0 +1,1274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Deferred: `mod self_hosted` is gated out (`#[cfg(any())]`) in `shm/mod.rs`. This file still +//! targets the removed `WorkerTransport`/`WorkerDispatch` dispatch umbrella, which the +//! `ChannelResolver`/`WorkerChannel` protocol has no analog for. Its no-gRPC-default role is now +//! served by `InProcessChannelResolver`, and the in-crate ring safety net by the `in_process` test, +//! so it stays unported until reimplemented as a `ChannelResolver` driving produce over +//! `coordinator_channel`. Kept for reference; it does not compile against this branch. +//! +//! The shared-memory transport hosting its own workers, so it can serve as a session default. +//! +//! Production embedders drive the mesh themselves: they allocate the region, launch the worker +//! processes, and deliver plans out of band, so [`ShmMqWorkerTransport`]'s dispatcher is a no-op. +//! That shape cannot be a default transport: a default gets nothing but the `WorkerTransport` +//! calls. [`SelfHostedShmTransport`] closes that gap by playing the embedder itself, in-process: +//! dispatch delivers each task's plan through the worker session machinery +//! ([`Worker::set_task_plan`]: codec round-trip, work-unit feed channels, metrics back-channel) +//! and runs the producer fragments as tasks pushing through a heap-backed DSM mesh; `open` reads +//! the rings from the leader side. Every cross-stage byte moves through the same rings, framing, +//! and cooperative drain a production embedder uses. +//! +//! Per query, the harness lives from the first dispatch to the cancellation token firing (the +//! head stream dropping). The mesh is sized and built lazily at the first `open`, once every +//! stage has been dispatched and the routing is known. + +use std::alloc::Layout; +use std::ffi::c_void; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use dashmap::DashMap; +use datafusion::common::instant::Instant; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{DataFusionError, HashMap, Result, exec_err, internal_err}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr_common::metrics::ExecutionPlanMetricsSet; +use datafusion::physical_plan::ExecutionPlan; +use futures::StreamExt; +use futures::stream::BoxStream; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use super::mpsc_ring::Wakeup; +use super::runtime::{MppMesh, ShmMqWorkerTransport, proc_for_task}; +use super::setup::{dsm_region_bytes, leader_setup, worker_setup}; +use super::transport::{ + CooperativeDrainSet, Interrupt, MppFrameHeader, MppPartitionSink, MppSender, SendBatchStats, + SetPlanFrame, +}; +use crate::proto as pb; +use crate::{ + CoordinatorToWorkerMetrics, DistributedTaskContext, MetricsStore, NetworkBoundaryExt, + NetworkCoalesceExec, PartitionSink, ProducerHead, RemoteStage, TreeNodeExt, Worker, + WorkerConnection, WorkerDispatch, WorkerDispatchRequest, WorkerSessionBuilder, WorkerTransport, + collect_task_work_unit_feeds, encode_task_plan, execute_local_task, + get_config_extension_propagation_headers, get_distributed_cancellation_token, + get_passthrough_headers, serialize_uuid, set_distributed_worker_transport, set_sent_time, +}; + +/// Per-inbox ring size. Frames fragment across slots, so this bounds a single batch at roughly +/// the whole ring; the suite's batches sit well under it while keeping the per-query footprint +/// (`n_procs` inboxes) modest. +const SELF_HOSTED_QUEUE_BYTES: usize = 1 << 22; + +/// No-op wakeup: the cooperative consumers yield instead of parking, so a publish never has to +/// wake a blocked thread. +pub(super) struct NoopWakeup; +impl Wakeup for NoopWakeup { + fn wake(&self, _token: u64) {} +} + +/// Opaque, non-sentinel receiver token. The wakeup ignores the value; this just keeps the +/// producer from treating the consumer as unregistered. +pub(super) fn receiver_token(proc_idx: u32) -> u64 { + proc_idx as u64 + 1 +} + +/// Owns the heap buffer standing in for a shared-memory segment. Every proc reads and writes the +/// same region through raw pointers; the lock-free rings make the concurrent access sound. +pub(super) struct HeapRegion { + ptr: *mut u8, + layout: Layout, +} + +impl HeapRegion { + pub(super) fn new(bytes: usize) -> Self { + // 64-byte alignment so each per-inbox ring header lands on its own cache line; the + // dsm layout aligns the offsets within the region, but only if the base is aligned too. + let layout = Layout::from_size_align(bytes, 64).expect("dsm region layout"); + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null(), "dsm region alloc failed"); + Self { ptr, layout } + } + + pub(super) fn base(&self) -> *mut c_void { + self.ptr as *mut c_void + } +} + +impl Drop for HeapRegion { + fn drop(&mut self) { + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } +} + +// The raw pointer is only dereferenced through the rings, which are designed for concurrent +// access from multiple procs. +unsafe impl Send for HeapRegion {} +unsafe impl Sync for HeapRegion {} + +/// Interrupt extension point wired to the query's cancellation token, so producers blocked in a send spin +/// and consumers in the pull loop both unwind when the head stream drops. +struct CancellationInterrupt(CancellationToken); +impl Interrupt for CancellationInterrupt { + fn check(&self) -> Result<(), DataFusionError> { + if self.0.is_cancelled() { + return Err(DataFusionError::Execution( + "mpp: query cancelled".to_string(), + )); + } + Ok(()) + } +} + +/// [`WorkerTransport`] over the shared-memory mesh, hosting its own workers in-process. +/// +/// All tasks share one [Worker] (one task registry, one session builder), like the in-memory +/// transport; what differs is the data plane: producer fragments run eagerly as background tasks +/// pushing through DSM rings, and reads pull from those rings instead of executing lazily. +/// +/// With the `flight` feature off this is the default transport. Multi-process embedders keep +/// driving [`ShmMqWorkerTransport`] directly. +#[derive(Clone)] +pub struct SelfHostedShmTransport { + worker: Worker, + queries: Arc>>, + queue_bytes: usize, +} + +impl Default for SelfHostedShmTransport { + fn default() -> Self { + Self::new(Worker::default()) + } +} + +impl SelfHostedShmTransport { + /// Builds the transport around an existing [Worker], sharing its task registry, session + /// builder, and runtime environment. + pub fn new(worker: Worker) -> Self { + Self { + worker, + queries: Arc::new(DashMap::new()), + queue_bytes: SELF_HOSTED_QUEUE_BYTES, + } + } + + /// Overrides the per-inbox ring size. Small values force multi-slot fragmentation and the + /// cooperative send spin on every query, which is how the ring mechanics get stress-tested; + /// the default is generous enough that only large batches touch them. + pub fn with_queue_bytes(mut self, queue_bytes: usize) -> Self { + self.queue_bytes = queue_bytes; + self + } + + /// Builds the transport with a custom [WorkerSessionBuilder], the same customization hook a + /// remote worker offers. + pub fn from_session_builder( + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> Self { + Self::new(Worker::from_session_builder(session_builder)) + } + + /// The in-process [Worker] backing this transport. + pub fn worker(&self) -> &Worker { + &self.worker + } +} + +/// Recovers the producer head the planner already inserted at the top of a stage's shipped +/// fragment. The worker's `insert_producer_head` strips an existing head and re-adds one from the +/// [`ProducerHead`] it is given, so a `None` here would flatten a sliced stage to one partition; +/// echoing the existing head back keeps it intact. +fn producer_head_from_plan(plan: &Arc) -> ProducerHead { + use datafusion::physical_plan::repartition::RepartitionExec; + if let Some(r) = plan.downcast_ref::() { + ProducerHead::RepartitionExec { + partitioning: r.partitioning().clone(), + } + } else if let Some(b) = plan.downcast_ref::() { + ProducerHead::BroadcastExec { + output_partitions: b.properties().partitioning.partition_count(), + } + } else { + ProducerHead::None + } +} + +impl WorkerTransport for SelfHostedShmTransport { + fn open( + &self, + input_stage: &RemoteStage, + target_partitions: std::ops::Range, + target_task: usize, + producer_head: ProducerHead, + ctx: &Arc, + metrics: &ExecutionPlanMetricsSet, + ) -> Result> { + let Some(harness) = self + .queries + .get(&input_stage.query_id) + .map(|e| Arc::clone(e.value())) + else { + return exec_err!( + "self-hosted shm transport: no harness for query {}; stage {} was never \ + dispatched through this transport", + input_stage.query_id, + input_stage.num + ); + }; + // The first read finalizes the harness: by now every stage has been dispatched (plan + // preparation completes before the head executes), so the mesh can be sized and the + // producer drivers released. + harness.ensure_started()?; + let leader_mesh = harness.leader_mesh()?; + let inner = ShmMqWorkerTransport::new(leader_mesh).open( + input_stage, + target_partitions, + target_task, + producer_head, + ctx, + metrics, + )?; + Ok(Box::new(PinnedConnection { inner, harness })) + } + + fn dispatcher(&self) -> Box { + Box::new(SelfHostedDispatcher { + transport: self.clone(), + metrics: OnceLock::new(), + }) + } +} + +/// Keeps the query harness (and through it the heap region the rings live in) alive for as long +/// as any stream is still reading from the mesh. +struct PinnedConnection { + inner: Box, + harness: Arc, +} + +impl WorkerConnection for PinnedConnection { + fn execute( + &self, + partition: usize, + ) -> Result>> { + let stream = self.inner.execute(partition)?; + let harness = Arc::clone(&self.harness); + Ok(Box::pin(stream.map(move |item| { + let _ = &harness; // <- the region must outlive the ring receivers. + item + }))) + } +} + +/// Per-query plan-delivery state. As with the other transports, the plan-send metrics and the +/// query start timestamp live for the whole query. +struct SelfHostedDispatcher { + transport: SelfHostedShmTransport, + metrics: OnceLock, +} + +impl WorkerDispatch for SelfHostedDispatcher { + fn dispatch(&self, request: WorkerDispatchRequest<'_>) -> Result<()> { + let WorkerDispatchRequest { + stage, + routed_urls, + task_ctx, + metrics, + metrics_store, + join_set, + .. + } = request; + let metrics = self + .metrics + .get_or_init(|| CoordinatorToWorkerMetrics::new(metrics)) + .clone(); + + let token = get_distributed_cancellation_token(task_ctx); + let harness = match self.transport.queries.entry(stage.query_id) { + dashmap::Entry::Occupied(e) => Arc::clone(e.get()), + dashmap::Entry::Vacant(e) => { + let harness = Arc::new(QueryHarness::new( + stage.query_id, + token.clone(), + self.transport.queue_bytes, + metrics_store.cloned(), + )); + e.insert(Arc::clone(&harness)); + // The token fires when the head stream drops (normal completion included), which + // is the query's end of life; drop the registry entry then. The region itself + // stays alive through the Arcs the drivers and streams hold. + let queries = Arc::clone(&self.transport.queries); + let query_id = stage.query_id; + let watched = token.clone(); + #[allow(clippy::disallowed_methods)] + tokio::spawn(async move { + watched.cancelled().await; + queries.remove(&query_id); + }); + // One pump set per query: drains every proc's inbox so control frames flow even + // when no consumer or producer is actively draining (a fragment blocked on its + // feed, the metrics frames after the gather finished), and forwards the task + // metrics into the store. Lives until every driver and feed pump reported done. + join_set.spawn(run_pumps(Arc::clone(&harness), token.clone())); + // Plans travel as `SetPlan` frames like every other control message, but the + // rings only exist at finalize; this pump waits for them and ships the queue. + harness.participants.fetch_add(1, Ordering::SeqCst); + join_set.spawn(run_plan_delivery(Arc::clone(&harness), token.clone())); + harness + } + }; + + let mut encoded_tasks = Vec::with_capacity(routed_urls.len()); + for task_i in 0..routed_urls.len() { + encoded_tasks.push(encode_task_plan( + &stage.plan, + task_i, + stage.tasks, + task_ctx.session_config(), + )?); + } + harness.record_stage( + stage.num, + stage.tasks, + encoded_tasks.iter().map(|e| e.partitions).collect(), + ); + harness + .state + .lock() + .unwrap() + .coord_metrics + .get_or_insert_with(|| metrics.clone()); + harness.scan_for_child_routing(&stage.plan, stage.tasks)?; + + // The planner already baked the producer head into the shipped fragment. Re-state it so + // each worker's `insert_producer_head` re-creates the same head rather than flattening the + // stage to one partition. + let producer_head = producer_head_from_plan(&stage.plan).to_proto(task_ctx)?; + + let mut headers = get_config_extension_propagation_headers(task_ctx.session_config())?; + headers.extend(get_passthrough_headers(task_ctx.session_config())); + + for (task_i, (url, encoded)) in routed_urls.iter().zip(encoded_tasks).enumerate() { + let plan_size = encoded.plan_proto.len(); + + let task_key = pb::TaskKey { + query_id: serialize_uuid(&stage.query_id), + stage_id: stage.num as u64, + task_number: task_i as u64, + }; + let set_plan = pb::SetPlanRequest { + plan_proto: encoded.plan_proto, + task_count: stage.tasks as u64, + task_key: Some(task_key.clone()), + work_unit_feed_declarations: encoded.feed_declarations, + target_worker_url: url.to_string(), + query_start_time_ns: metrics.instantiation_time, + }; + // The plan reaches the driver as a `SetPlan` frame on its proc's inbox, the same + // wire crossing a separate-process worker would see; the driver only holds its + // address. The frames queue here because the rings are only built at finalize. + harness + .state + .lock() + .unwrap() + .pending_set_plans + .push(PendingSetPlan { + stage_num: stage.num as u32, + task_i, + frame: SetPlanFrame::from_parts(set_plan, &headers)?, + plan_size, + }); + + // Collected before spawning so the providers see the same eager `feed()` timing as + // they do under the other transports. + let feed_streams = + collect_task_work_unit_feeds(&stage.plan, task_ctx, task_i, stage.tasks)?; + let has_feeds = !feed_streams.is_empty(); + + let driver = TaskDriver { + harness: Arc::clone(&harness), + worker: self.transport.worker.clone(), + stage_num: stage.num as u32, + task_i, + n_partitions: encoded.partitions, + has_feeds, + task_key, + token: token.clone(), + producer_head: producer_head.clone(), + }; + harness.participants.fetch_add(1, Ordering::SeqCst); + join_set.spawn(driver.run()); + + // The feeds travel as `WorkUnit` frames through the leader's outbound senders, so + // the worker side reads them exactly as a separate process would. + if has_feeds { + harness.state.lock().unwrap().any_feeds = true; + harness.participants.fetch_add(1, Ordering::SeqCst); + join_set.spawn(run_leader_feed_pump( + Arc::clone(&harness), + stage.num as u32, + task_i, + feed_streams, + token.clone(), + )); + } + } + Ok(()) + } +} + +/// How a producer stage's output reaches its consumers, as parent-stage task indexes. A stage no +/// parent boundary ever claims has no entry: it is consumed by the head on the leader (proc 0). +/// +/// Built by simulating each consumer task's reads under its effective task contexts (a +/// `ChildrenIsolatorUnionExec` hands its children remapped contexts, so a boundary under one is +/// read with that inner context, not the stage-level one). `None` slots were never claimed by +/// any consumer (padding partitions); they route to the leader, where they sit buffered until +/// teardown. +enum RoutingSpec { + /// Nested `NetworkCoalesceExec`: consumers read whole producer tasks, so the destination + /// depends on the producer task only. + PerTask(Vec>), + /// Nested shuffle/broadcast: consumers read partition slices, identical across producer + /// tasks, so the destination depends on the output partition only. + PerPartition(Vec>), +} + +struct StageRec { + tasks: usize, + /// Output partitions of each task's specialized plan. Task-isolated nodes make these differ + /// from the unspecialized stage plan (and possibly from each other). + task_partitions: Vec, +} + +/// What a task driver needs to start producing: its proc's mesh and one routed sink per output +/// partition. +struct Launch { + mesh: Arc, + sinks: Vec>, + /// Send end for this task's `TaskMetrics` frame to the leader. + metrics_sender: MppSender, +} + +/// One task's plan, queued between `dispatch` and finalize, when the rings exist to carry it. +struct PendingSetPlan { + stage_num: u32, + task_i: usize, + frame: SetPlanFrame, + plan_size: usize, +} + +struct HarnessState { + stages: HashMap, + routing: HashMap, + started: bool, + /// Whether any dispatched task declared work-unit feeds; decides whether a feed pump runs. + any_feeds: bool, + /// Plans queued by `dispatch` until finalize builds the mesh; the delivery pump then ships + /// each as a `SetPlan` frame to the proc hosting its task. + pending_set_plans: Vec, + /// Dispatch-side metrics (plan send latency / bytes), recorded by the delivery pump. + coord_metrics: Option, + n_workers: u32, + leader_mesh: Option>, + /// The leader's outbound senders, the control-plane path for `WorkUnit` frames. + leader_senders: Vec>, + /// One mesh per worker proc, kept for the per-proc drain pumps. + worker_meshes: Vec>, + launches: HashMap<(u32, usize), Launch>, + /// Declared after the meshes so it would drop last either way; the harness Arcs held by + /// drivers and pinned streams are what actually keep it alive long enough. + region: Option, +} + +struct QueryHarness { + query_id: Uuid, + token: CancellationToken, + queue_bytes: usize, + metrics_store: Option>, + /// Drivers and feed pumps spawned for this query; `done` counts their exits. The pumps run + /// until the two meet, which is the deterministic "no more frames are coming" signal. + participants: AtomicUsize, + done: AtomicUsize, + state: Mutex, + ready_tx: tokio::sync::watch::Sender, + ready_rx: tokio::sync::watch::Receiver, +} + +impl QueryHarness { + fn new( + query_id: Uuid, + token: CancellationToken, + queue_bytes: usize, + metrics_store: Option>, + ) -> Self { + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); + Self { + query_id, + token, + queue_bytes, + metrics_store, + participants: AtomicUsize::new(0), + done: AtomicUsize::new(0), + state: Mutex::new(HarnessState { + stages: HashMap::new(), + routing: HashMap::new(), + started: false, + any_feeds: false, + pending_set_plans: Vec::new(), + coord_metrics: None, + n_workers: 0, + leader_mesh: None, + leader_senders: Vec::new(), + worker_meshes: Vec::new(), + launches: HashMap::new(), + region: None, + }), + ready_tx, + ready_rx, + } + } + + /// Wait until the first read finalizes the harness. + async fn ready(&self) -> Result<()> { + let mut rx = self.ready_rx.clone(); + while !*rx.borrow() { + rx.changed().await.map_err(|_| { + DataFusionError::Internal( + "self-hosted shm transport: harness dropped before start".to_string(), + ) + })?; + } + Ok(()) + } + + /// The leader's send end for one task's `WorkUnit` frames, cooperative-draining the + /// leader's own inbox so a symmetric full-ring stall cannot deadlock the feed push. + fn leader_feed_sender(&self, stage_num: u32, task_i: usize) -> Result { + let state = self.state.lock().unwrap(); + let dest_proc = proc_for_task(state.n_workers, task_i as u32); + let Some(base) = state + .leader_senders + .get(dest_proc as usize) + .and_then(|s| s.as_ref()) + else { + return internal_err!( + "self-hosted shm transport: no leader sender for proc {dest_proc}" + ); + }; + let Some(leader_mesh) = state.leader_mesh.clone() else { + return internal_err!("self-hosted shm transport: leader mesh not built"); + }; + Ok(base + .clone_with_header(MppFrameHeader::work_unit(stage_num, task_i as u32, 0)) + .with_cooperative_drain(leader_mesh as Arc)) + } + + fn record_stage(&self, num: usize, tasks: usize, task_partitions: Vec) { + let mut state = self.state.lock().unwrap(); + state.stages.insert( + num as u32, + StageRec { + tasks, + task_partitions, + }, + ); + } + + /// Classify the routing of every child stage referenced by `plan`'s network boundaries. The + /// children were dispatched before this stage (plan preparation converts bottom-up), so their + /// records exist; stages no parent ever claims are consumed by the head on the leader. + fn scan_for_child_routing( + &self, + plan: &Arc, + consumer_tasks: usize, + ) -> Result<()> { + let mut state = self.state.lock().unwrap(); + for task_i in 0..consumer_tasks { + let d_ctx = DistributedTaskContext { + task_index: task_i, + task_count: consumer_tasks, + }; + let state = &mut *state; + let mut scan_err = Ok(()); + plan.apply_with_dt_ctx(d_ctx, |node, d_ctx| { + let Some(nb) = node.as_ref().as_network_boundary() else { + return Ok(TreeNodeRecursion::Continue); + }; + let child_num = nb.input_stage().num() as u32; + let Some(rec) = state.stages.get(&child_num) else { + scan_err = internal_err!( + "self-hosted shm transport: stage {child_num} referenced before dispatch" + ); + return Ok(TreeNodeRecursion::Stop); + }; + let child_tasks = rec.tasks; + let child_max_parts = rec.task_partitions.iter().copied().max().unwrap_or(0); + + if node.as_ref().is::() { + let spec = state + .routing + .entry(child_num) + .or_insert_with(|| RoutingSpec::PerTask(vec![None; child_tasks])); + let RoutingSpec::PerTask(dest) = spec else { + scan_err = internal_err!( + "self-hosted shm transport: stage {child_num} read through mixed \ + boundary kinds" + ); + return Ok(TreeNodeRecursion::Stop); + }; + // Mirror of the consumer's `task_group` split: contiguous groups, the first + // `extra` groups one producer task longer. + let base = child_tasks / d_ctx.task_count.max(1); + let extra = child_tasks % d_ctx.task_count.max(1); + let len = base + usize::from(d_ctx.task_index < extra); + let start = d_ctx.task_index * base + d_ctx.task_index.min(extra); + let end = (start + len).min(child_tasks); + for slot in dest[start..end].iter_mut() { + *slot = Some(task_i as u32); + } + } else { + let spec = state + .routing + .entry(child_num) + .or_insert_with(|| RoutingSpec::PerPartition(vec![None; child_max_parts])); + let RoutingSpec::PerPartition(dest) = spec else { + scan_err = internal_err!( + "self-hosted shm transport: stage {child_num} read through mixed \ + boundary kinds" + ); + return Ok(TreeNodeRecursion::Stop); + }; + // Shuffle and broadcast read the same partition slice per consumer context. + let p_c = nb.partitions_per_consumer_task(); + let from = (p_c * d_ctx.task_index).min(child_max_parts); + let to = (p_c * (d_ctx.task_index + 1)).min(child_max_parts); + for slot in dest[from..to].iter_mut() { + *slot = Some(task_i as u32); + } + } + Ok(TreeNodeRecursion::Continue) + })?; + scan_err?; + } + Ok(()) + } + + /// Size and build the mesh, resolve the routing, and release the waiting task drivers. Runs + /// once, on the first `open`. + fn ensure_started(&self) -> Result<()> { + let mut state = self.state.lock().unwrap(); + if state.started { + return Ok(()); + } + + let n_workers = state + .stages + .values() + .map(|s| s.tasks) + .max() + .unwrap_or(1) + .max(1) as u32; + let n_procs = n_workers + 1; + + let region_total = dsm_region_bytes(n_procs, self.queue_bytes, 0)?; + let region = HeapRegion::new(region_total); + let wakeup: Arc = Arc::new(NoopWakeup); + let interrupt: Arc = Arc::new(CancellationInterrupt(self.token.clone())); + + let leader_attach = unsafe { + leader_setup( + region.base(), + n_procs, + self.queue_bytes, + &[], + Arc::clone(&wakeup), + receiver_token(0), + Arc::clone(&interrupt), + // The harness holds the senders until the query ends, so attaching is safe, + // and the plan delivery pump always needs them. + true, + ) + }?; + let leader_mesh = leader_attach.mesh; + let mut worker_meshes = Vec::with_capacity(n_workers as usize); + for proc_idx in 1..n_procs { + let attach = unsafe { + worker_setup( + region.base(), + region_total, + proc_idx, + Arc::clone(&wakeup), + receiver_token(proc_idx), + Arc::clone(&interrupt), + ) + }?; + worker_meshes.push((attach.mesh, attach.outbound_senders)); + } + + // Build every fragment's launch package: one routed sink per output partition, sharing + // the proc's outbound senders. The base senders drop at the end of this scope, so the + // rings observe the last-sender detach once the per-partition clones finish. + let mut launches = HashMap::new(); + for (&stage_num, rec) in state.stages.iter() { + let spec = state.routing.get(&stage_num); + for task_i in 0..rec.tasks { + let n_out = rec.task_partitions.get(task_i).copied().unwrap_or(0); + let proc = proc_for_task(n_workers, task_i as u32); + let (mesh, outbound) = &worker_meshes[(proc - 1) as usize]; + let Some(to_leader) = outbound[0].as_ref() else { + return internal_err!( + "self-hosted shm transport: no outbound sender from proc {proc} to the \ + leader" + ); + }; + // No cooperative drain on purpose: metrics frames go out after the cancellation + // token fired (it fires on normal completion), when the interrupt-checking spin + // would abort the send. + let metrics_sender = to_leader.clone_with_header(MppFrameHeader::task_metrics( + stage_num, + task_i as u32, + proc, + )); + let mut sinks: Vec> = Vec::with_capacity(n_out); + for q in 0..n_out { + let consumer = match spec { + // No parent boundary claimed this stage: the head consumes it on the + // leader. + None => None, + Some(RoutingSpec::PerTask(dest)) => dest.get(task_i).copied().flatten(), + Some(RoutingSpec::PerPartition(dest)) => dest.get(q).copied().flatten(), + }; + let dest_proc = match (spec, consumer) { + (None, _) | (_, None) => 0, + (_, Some(parent_task)) => proc_for_task(n_workers, parent_task), + }; + let Some(base) = outbound[dest_proc as usize].as_ref() else { + return internal_err!( + "self-hosted shm transport: no outbound sender from proc {proc} to \ + proc {dest_proc}" + ); + }; + let sender = base + .clone_with_header(MppFrameHeader::batch(stage_num, q as u32, proc)) + .with_cooperative_drain(Arc::clone(mesh) as Arc); + sinks.push(Box::new(MppPartitionSink::new(sender))); + } + launches.insert( + (stage_num, task_i), + Launch { + mesh: Arc::clone(mesh), + sinks, + metrics_sender, + }, + ); + } + } + + state.n_workers = n_workers; + state.leader_mesh = Some(leader_mesh); + state.leader_senders = leader_attach.outbound_senders; + state.worker_meshes = worker_meshes.iter().map(|(m, _)| Arc::clone(m)).collect(); + state.launches = launches; + state.region = Some(region); + state.started = true; + drop(state); + let _ = self.ready_tx.send(true); + Ok(()) + } + + fn leader_mesh(&self) -> Result> { + self.state + .lock() + .unwrap() + .leader_mesh + .clone() + .ok_or_else(|| { + DataFusionError::Internal( + "self-hosted shm transport: leader mesh not built".to_string(), + ) + }) + } + + /// Wait until the harness is finalized, then take this fragment's launch package. + async fn wait_launch(&self, stage_num: u32, task_i: usize) -> Result { + self.ready().await?; + let mut state = self.state.lock().unwrap(); + state.launches.remove(&(stage_num, task_i)).ok_or_else(|| { + DataFusionError::Internal(format!( + "self-hosted shm transport: no launch package for stage {stage_num} task {task_i}" + )) + }) + } +} + +/// Delivers one task's plan through the worker session machinery and produces its fragment into +/// the mesh. +struct TaskDriver { + harness: Arc, + worker: Worker, + stage_num: u32, + task_i: usize, + n_partitions: usize, + has_feeds: bool, + task_key: pb::TaskKey, + token: CancellationToken, + /// The producer head the distributed planner already baked into the shipped fragment. + /// Re-stated here so the worker's `insert_producer_head` strips and re-adds the same head + /// (a no-op) instead of flattening the stage to a single partition. + producer_head: pb::execute_task_request::ProducerHead, +} + +impl TaskDriver { + async fn run(self) -> Result<()> { + let harness = Arc::clone(&self.harness); + let result = self.run_inner().await; + harness.done.fetch_add(1, Ordering::SeqCst); + result + } + + async fn run_inner(self) -> Result<()> { + let Self { + harness, + worker, + stage_num, + task_i, + n_partitions, + has_feeds, + task_key, + token, + producer_head, + } = self; + + // The launch package arrives when the first read finalizes the harness. A query torn + // down before any read just unwinds the driver. + let launch = tokio::select! { + launch = harness.wait_launch(stage_num, task_i) => launch?, + _ = token.cancelled() => return Ok(()), + }; + + // The plan arrives as a `SetPlan` frame on this proc's inbox, shipped by the leader's + // delivery pump; the pumps drain it into the registry this take resolves from. + let set_plan_frame = tokio::select! { + frame = launch.mesh.take_set_plan(stage_num, task_i as u32) => frame?, + _ = token.cancelled() => return Ok(()), + }; + let (set_plan, headers) = set_plan_frame.into_parts()?; + + let mesh = Arc::clone(&launch.mesh); + let outcome = worker + .set_task_plan(set_plan, headers, move |mut cfg| { + // Child-stage reads inside the decoded fragment must pull from this proc's + // inbox, and its dispatcher must stay a no-op: the plans are already here. + set_distributed_worker_transport(&mut cfg, ShmMqWorkerTransport::new(mesh)); + Ok(cfg) + }) + .await?; + + // The feeds arrive as `WorkUnit` frames on this proc's inbox; install the channels so + // the drain fills them (and flushes whatever arrived first). The leader-side pump owns + // delivery and the `FeedEof` that ends the streams. + if has_feeds { + launch.mesh.register_work_unit_senders( + stage_num, + task_i as u32, + outcome.work_unit_senders, + ); + } + + let produce = async { + let request = pb::ExecuteTaskRequest { + task_key: Some(task_key), + target_partition_start: 0, + target_partition_end: n_partitions as u64, + producer_head: Some(producer_head), + }; + // Through `execute_local_task` rather than a bare `plan.execute` so the task metrics + // (added/executed/finished stamps, per-node metrics) flow exactly like a pull-based + // read would deliver them. + let (streams, _ctx) = execute_local_task(worker.task_data_entries(), request).await?; + if streams.len() != launch.sinks.len() { + return internal_err!( + "self-hosted shm transport: stage {stage_num} task {task_i} decoded into {} \ + partitions but routed {} sinks", + streams.len(), + launch.sinks.len() + ); + } + let mut futures = Vec::with_capacity(streams.len()); + for (mut stream, mut sink) in streams.into_iter().zip(launch.sinks) { + let token = token.clone(); + futures.push(async move { + let stream_result: Result<()> = async { + loop { + let batch = tokio::select! { + next = stream.next() => next, + // Head stream dropped: stop pulling so this fragment and its upstream + // scan unwind, instead of draining the input into a buffer no one reads. + _ = token.cancelled() => break, + }; + let Some(batch) = batch else { break }; + let batch = batch?; + if batch.num_rows() == 0 { + continue; + } + sink.send(&batch).await?; + // A downstream worker abandoned this stream (its mesh carries the cancel + // senders): stop pulling so the cancel cascades to this fragment's own + // producers, matching the embedder's `run_worker_fragment` loop. + if sink.cancelled() { + break; + } + } + Ok(()) + } + .await; + // EOF always, even after a failed send, so the consumer side unblocks; the + // stream error stays the primary one. + let eof_result = sink.finish().await; + stream_result.and(eof_result) + }); + } + // `join_all`, not fail-fast: cancelling sibling partitions mid-await would skip their + // EOFs and wedge the consumer's channel buffers. + let results = futures::future::join_all(futures).await; + for r in results { + r?; + } + Ok(()) + }; + let produce_res: Result<()> = produce.await; + + // The metrics receiver resolves as the last partition stream above completes, so this + // does not block on anything remote. The frame goes back over the mesh, where the + // leader-side pump forwards it into the metrics store. + if let Ok(task_metrics) = outcome.metrics_rx.await { + let _ = launch + .metrics_sender + .send_task_metrics_best_effort(&task_metrics) + .await; + } + produce_res + } +} + +/// Leader-side delivery of every dispatched plan: waits for finalize to build the rings, then +/// ships each queued plan as a `SetPlan` frame to the proc hosting its task. One pump per query, +/// counted as a participant so the drain pumps outlive it. +async fn run_plan_delivery(harness: Arc, token: CancellationToken) -> Result<()> { + let result = run_plan_delivery_inner(&harness, token).await; + harness.done.fetch_add(1, Ordering::SeqCst); + result +} + +async fn run_plan_delivery_inner( + harness: &Arc, + token: CancellationToken, +) -> Result<()> { + tokio::select! { + ready = harness.ready() => ready?, + _ = token.cancelled() => return Ok(()), + } + let (pending, senders, metrics) = { + let mut state = harness.state.lock().unwrap(); + let pending = std::mem::take(&mut state.pending_set_plans); + let Some(leader_mesh) = state.leader_mesh.clone() else { + return internal_err!("self-hosted shm transport: leader mesh not built"); + }; + let mut senders = Vec::with_capacity(pending.len()); + for plan in &pending { + let dest_proc = proc_for_task(state.n_workers, plan.task_i as u32); + let Some(base) = state + .leader_senders + .get(dest_proc as usize) + .and_then(|s| s.as_ref()) + else { + return internal_err!( + "self-hosted shm transport: no leader sender for proc {dest_proc}" + ); + }; + senders.push( + base.clone_with_header(MppFrameHeader::set_plan( + plan.stage_num, + plan.task_i as u32, + 0, + )) + .with_cooperative_drain(Arc::clone(&leader_mesh) as Arc), + ); + } + (pending, senders, state.coord_metrics.clone()) + }; + let mut stats = SendBatchStats::default(); + for (plan, sender) in pending.into_iter().zip(senders) { + let start = Instant::now(); + sender.send_set_plan_traced(&plan.frame, &mut stats).await?; + if let Some(metrics) = &metrics { + metrics.plan_send_latency.record(&start); + metrics.plan_bytes_sent.add_bytes(plan.plan_size); + } + } + Ok(()) +} + +/// Leader-side pump for one task's work-unit feeds: drives the providers and ships each unit as +/// a `WorkUnit` frame to the proc hosting the task, then closes the task's channels with a +/// `FeedEof`. The close is unconditional, error or not: without it the fragment's feed streams +/// never end and the worker side wedges instead of surfacing this pump's error. +async fn run_leader_feed_pump( + harness: Arc, + stage_num: u32, + task_i: usize, + feed_streams: Vec>>, + token: CancellationToken, +) -> Result<()> { + let result = async { + tokio::select! { + ready = harness.ready() => ready?, + _ = token.cancelled() => return Ok(()), + } + let sender = harness.leader_feed_sender(stage_num, task_i)?; + + let pump_result = async { + let mut pumps = Vec::with_capacity(feed_streams.len()); + for mut stream in feed_streams { + let sender = &sender; + pumps.push(async move { + let mut stats = SendBatchStats::default(); + while let Some(unit) = stream.next().await { + let mut unit = unit?; + set_sent_time(&mut unit); + sender.send_work_unit_traced(&unit, &mut stats).await?; + } + Ok::<_, DataFusionError>(()) + }); + } + futures::future::try_join_all(pumps).await.map(|_| ()) + } + .await; + + let mut stats = SendBatchStats::default(); + let eof_result = sender.send_feed_eof_traced(&mut stats).await; + pump_result.and(eof_result) + } + .await; + harness.done.fetch_add(1, Ordering::SeqCst); + result +} + +/// Per-proc drain pumps plus the metrics forwarder, alive until every driver and feed pump +/// reported done. They are what moves control frames when nothing else drains: a fragment +/// blocked on its feed before producing, and the metrics frames arriving after the gather +/// already finished. +async fn run_pumps(harness: Arc, token: CancellationToken) -> Result<()> { + tokio::select! { + ready = harness.ready() => ready?, + _ = token.cancelled() => return Ok(()), + } + let (leader_mesh, worker_meshes) = { + let state = harness.state.lock().unwrap(); + let Some(leader_mesh) = state.leader_mesh.clone() else { + return internal_err!("self-hosted shm transport: leader mesh not built"); + }; + (leader_mesh, state.worker_meshes.clone()) + }; + let metrics_rx = leader_mesh.take_task_metrics_receiver(); + + let mut meshes: Vec> = Vec::with_capacity(worker_meshes.len() + 1); + meshes.push(Arc::clone(&leader_mesh)); + meshes.extend(worker_meshes); + let pumps = meshes.into_iter().map(|mesh| { + let harness = Arc::clone(&harness); + async move { + loop { + let all_done = harness.done.load(Ordering::SeqCst) + >= harness.participants.load(Ordering::SeqCst); + // Drain errors surface through the consumers reading the same registry; + // the pump just stops contributing. + if mesh.try_drain_pass().is_err() { + break; + } + if all_done { + // The pass above ran after the last participant reported done, so every + // frame sent before that point has been routed. + break; + } + tokio::task::yield_now().await; + } + } + }); + futures::future::join_all(pumps).await; + + // Every frame is routed by now; whatever metrics arrived are in the channel. + if let (Some(mut rx), Some(store)) = (metrics_rx, harness.metrics_store.clone()) { + while let Ok((stage_id, task_number, task_metrics)) = rx.try_recv() { + store.insert( + pb::TaskKey { + query_id: serialize_uuid(&harness.query_id), + stage_id: stage_id as u64, + task_number: task_number as u64, + }, + task_metrics, + ); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; + use crate::test_utils::session_context::register_temp_parquet_table; + use crate::{DistributedConfig, DistributedExt, SessionStateBuilderExt, display_plan_ascii}; + use datafusion::arrow::array::{Int32Array, StringArray}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::execute_stream; + use datafusion::prelude::SessionContext; + use futures::TryStreamExt; + + /// Forces the ring mechanics on every batch: with `RING_SLOTS = 8`, a 64 KiB inbox has + /// ~8 KiB slots, so the ~16 KiB frames below fragment across slots, and the ~2 MB of + /// payload wraps each ring dozens of times, exercising the cooperative send spin. + const TINY_QUEUE_BYTES: usize = 64 * 1024; + + const ROWS: usize = 2000; + + fn sample_batch() -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("s", DataType::Utf8, false), + Field::new("val", DataType::Int32, false), + ])); + // ~1 KiB per row, unique values so the GROUP BY keeps the full volume flowing + // through the shuffle instead of compacting it away at the partial aggregate. + let strings: Vec = (0..ROWS) + .map(|i| format!("{i:06}-{}", "x".repeat(1024))) + .collect(); + let vals: Vec = (0..ROWS as i32).collect(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(strings)), + Arc::new(Int32Array::from(vals)), + ], + ) + .unwrap() + } + + async fn run(ctx: &SessionContext) -> Result<(String, Vec)> { + // Shipping `s` itself out of the aggregate is the point of this query: each emitted + // batch is an offset slice of the aggregate's accumulated state, and arrow-ipc + // serializes the slice's whole backing buffer, so the raw frames dwarf these tiny + // rings. The query only completes if the send path compacts and chunks them. + let query = "SELECT val, max(s) AS m FROM t GROUP BY val"; + let plan = ctx.sql(query).await?.create_physical_plan().await?; + let display = display_plan_ascii(plan.as_ref(), false); + let batches: Vec<_> = execute_stream(plan, ctx.task_ctx())?.try_collect().await?; + let mut lines: Vec = pretty_format_batches(&batches)? + .to_string() + .lines() + .map(str::to_string) + .collect(); + lines.sort(); + Ok((display, lines)) + } + + /// A high-cardinality shuffle query over rings far smaller than the data, so every + /// cross-stage byte moves through fragmented frames under send-spin backpressure. The + /// result must still match the serial reference exactly. + #[tokio::test] + async fn tiny_rings_force_fragmentation_and_backpressure() -> Result<()> { + let transport = SelfHostedShmTransport::default().with_queue_bytes(TINY_QUEUE_BYTES); + // Small producer batches keep each frame a few slots big instead of overflowing the + // whole ring (a single frame must fit within one ring). + let d_cfg = DistributedConfig { + shuffle_batch_size: 16, + ..Default::default() + }; + let mut state = SessionStateBuilder::new() + .with_default_features() + .with_distributed_option_extension(d_cfg) + .with_distributed_planner() + .with_distributed_task_estimator(2) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + .with_distributed_worker_transport(transport) + .build(); + state.config_mut().options_mut().execution.target_partitions = 3; + let ctx = SessionContext::from(state); + let path = + register_temp_parquet_table("t", sample_batch().schema(), vec![sample_batch()], &ctx) + .await?; + + let (display, distributed) = run(&ctx).await?; + assert!( + display.contains("NetworkShuffleExec"), + "the query did not distribute:\n{display}" + ); + + let single = SessionContext::default(); + single + .register_parquet("t", path.to_string_lossy().as_ref(), Default::default()) + .await?; + let (_, expected) = run(&single).await?; + + assert_eq!(distributed, expected); + Ok(()) + } + + /// The per-query harness must be reclaimed when the output stream drops, or the `queries` map + /// grows one entry per query for the transport's lifetime. + #[tokio::test] + async fn query_harness_is_reclaimed_after_the_stream_drops() -> Result<()> { + let transport = SelfHostedShmTransport::default(); + let probe = transport.clone(); + let mut state = SessionStateBuilder::new() + .with_default_features() + .with_distributed_planner() + .with_distributed_task_estimator(2) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + .with_distributed_worker_transport(transport) + .build(); + state.config_mut().options_mut().execution.target_partitions = 3; + let ctx = SessionContext::from(state); + register_temp_parquet_table("t", sample_batch().schema(), vec![sample_batch()], &ctx) + .await?; + + let (display, _) = run(&ctx).await?; + assert!( + display.contains("NetworkShuffleExec"), + "the query did not distribute:\n{display}" + ); + + // The token fires as the gathered stream drops above; the registry entry is dropped on a + // task the cancelled token wakes, so poll briefly rather than assume it already ran. + for _ in 0..100 { + if probe.queries.is_empty() { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!( + "queries map still holds {} entries; the harness leaked", + probe.queries.len() + ); + } +} diff --git a/src/shm/setup.rs b/src/shm/setup.rs new file mode 100644 index 000000000..d630c63cb --- /dev/null +++ b/src/shm/setup.rs @@ -0,0 +1,373 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Mesh construction over a caller-supplied shared buffer, the extension point between an embedder's buffer +//! allocation and the transport. +//! +//! The embedder allocates the shared region (a PG `dsm_segment`, or a heap buffer in-process), +//! sizes it with [`dsm_region_bytes`], then calls [`leader_setup`] on the proc that initializes the +//! rings and [`worker_setup`] on each producer proc. Both hand back an [`MppMesh`] the embedder +//! installs on its DataFusion session; `worker_setup` also returns the outbound senders and the +//! plan bytes copied out of the region. [`run_worker_fragment`] is the producer push loop. +//! +//! The two platform primitives the embedder supplies are the [`Wakeup`] (how to wake a blocked +//! consumer) and the [`Interrupt`] (how to check for cancellation); everything here is otherwise +//! pure Rust over the shared buffer. + +use std::ffi::c_void; +use std::sync::Arc; + +use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use futures::stream::StreamExt; + +use super::dsm::{ + compute_dsm_layout, leader_init, peer_proc_for_index, read_region_total, worker_attach, +}; +use super::mesh::{DsmInboxReceiver, DsmInboxSender}; +use super::mpsc_ring::{DsmMpscSender, NO_RECEIVER_TOKEN, Wakeup}; +use super::runtime::MppMesh; +use super::transport::{ + BatchChannelSender, DrainHandle, Interrupt, MppFrameHeader, MppReceiver, MppSender, + ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, +}; +use crate::proto as pb; +use crate::work_unit_feed::RemoteWorkUnitFeedRegistry; +use crate::{DistributedTaskContext, PartitionSink, collect_plan_metrics_protos}; + +/// Total bytes the shared region needs for `n_procs` inboxes plus `plan_len` plan bytes, with +/// `queue_bytes` per inbox. The embedder reserves exactly this much before [`leader_setup`]. +pub fn dsm_region_bytes(n_procs: u32, queue_bytes: usize, plan_len: usize) -> Result { + compute_dsm_layout(n_procs, queue_bytes, plan_len) + .map(|l| l.region_total) + .map_err(|e| DataFusionError::Internal(format!("mpp: dsm_region_bytes: {e}"))) +} + +/// Read `region_total` out of the header a leader wrote, so a worker that just mapped the region +/// can size its [`worker_setup`] call without knowing the header layout. +/// +/// A caller that derives the size from the header forfeits [`worker_setup`]'s size validation +/// (it would compare the header against itself). Pass the mapped size from the embedder's own +/// bookkeeping where it is available. +/// +/// # Safety +/// - `base` must point at the start of a region a leader initialized via [`leader_setup`]. +/// - `base` must be at least 8-byte aligned (the header holds `u64` fields). +pub unsafe fn region_total(base: *const c_void) -> usize { + unsafe { read_region_total(base) as usize } +} + +/// Wrap each peer-indexed `DsmMpscSender` into an outbound `MppSender` keyed by destination proc. +/// The dispatcher `clone_with_header`s these per output partition before sending, so the +/// placeholder header is never observed on the wire. Slot `this_proc` stays `None` until the +/// self-loop install. +/// +/// Returns `(data, cancel)`. `data` is the producer's output senders. `cancel` is a control-plane +/// sibling onto each peer inbox, used by [`MppMesh::cancel_stream`]: a consumer reaches its producer +/// without counting as one of that producer's data senders, so a held `Cancel` sender never masks +/// the producer-gone `detached` signal. +fn build_outbound_senders( + this_proc: u32, + total_procs: u32, + peer_senders: Vec, +) -> (Vec>, Vec>) { + let mut senders: Vec> = (0..total_procs).map(|_| None).collect(); + let mut cancel: Vec> = (0..total_procs).map(|_| None).collect(); + for (peer_idx, dsm_send) in peer_senders.into_iter().enumerate() { + let target_proc = peer_proc_for_index(this_proc, peer_idx as u32); + // A `peer_proc_for_index` regression that maps a peer onto the self slot would be + // silently overwritten by the self-loop install and only surface later as a missing + // sender at dispatch; name the bug at its source. + debug_assert!( + target_proc != this_proc, + "peer index {peer_idx} mapped to the self proc {this_proc}" + ); + debug_assert!( + target_proc < total_procs, + "peer index {peer_idx} mapped to proc {target_proc} >= total {total_procs}" + ); + let control: Arc = + Arc::new(DsmInboxSender::new(dsm_send.to_control())); + cancel[target_proc as usize] = Some(MppSender::with_header( + control, + MppFrameHeader::batch(0, 0, this_proc), + )); + let shared: Arc = Arc::new(DsmInboxSender::new(dsm_send)); + senders[target_proc as usize] = Some(MppSender::with_header( + shared, + // Stamp `sender_proc = this_proc` so a stray frame that escapes the dispatcher's + // `clone_with_header` overwrite still identifies its origin on the drain side. + MppFrameHeader::batch(0, 0, this_proc), + )); + } + (senders, cancel) +} + +/// What [`leader_setup`] hands back to the embedder. +pub struct LeaderAttach { + /// The leader's mesh, installed on its DataFusion session. + pub mesh: Arc, + /// Outbound senders keyed by destination proc index, for the control plane: work-unit + /// frames flow leader -> worker through them. Slot 0 (the leader itself) stays `None`; + /// empty unless `attach_senders` was passed. Holders must keep them alive for the whole + /// query: dropping them before a worker attaches latches that worker's inbox as detached. + pub outbound_senders: Vec>, +} + +/// Initialize the shared region as the leader (`proc 0`) and return its mesh plus its outbound +/// senders. +/// +/// Writes the region header, copies `plan_bytes` in, initializes the `n_procs` inboxes, and +/// attaches the leader as receiver to its own inbox. `receiver_token` is registered so producers +/// resolve this proc's [`Wakeup`]; `interrupt` is consulted at the transport's block points. +/// +/// # Safety +/// - `base` must point at an uninitialized region of at least `dsm_region_bytes(n_procs, +/// queue_bytes, plan_bytes.len())` bytes. +/// - `base` must be at least 8-byte (MAXALIGN) aligned; the ring headers hold atomics. +/// - The region must not be concurrently accessed until this returns. +#[allow(clippy::too_many_arguments)] // mirrors worker_setup; the args are the embedder's knobs +pub unsafe fn leader_setup( + base: *mut c_void, + n_procs: u32, + queue_bytes: usize, + plan_bytes: &[u8], + wakeup: Arc, + receiver_token: u64, + interrupt: Arc, + attach_senders: bool, +) -> Result { + if receiver_token == NO_RECEIVER_TOKEN { + return Err(DataFusionError::Internal( + "mpp: leader_setup: receiver_token is the NO_RECEIVER_TOKEN sentinel; wakeups \ + for this proc would be silently disabled" + .into(), + )); + } + let layout = compute_dsm_layout(n_procs, queue_bytes, plan_bytes.len()) + .map_err(|e| DataFusionError::Internal(format!("mpp: leader_setup compute layout: {e}")))?; + let attach = unsafe { + leader_init( + base, + &layout, + plan_bytes, + Arc::clone(&wakeup), + attach_senders, + ) + } + .map_err(DataFusionError::Internal)?; + + let inbox = DsmInboxReceiver::new(attach.inbound_receiver); + inbox.set_receiver(receiver_token); + let inbound = Arc::new(DrainHandle::cooperative( + 0, + vec![(ReceiverScope::Inbox, MppReceiver::new(Box::new(inbox)))], + )); + // The leader hosts no producer fragments, but its senders carry the control plane: + // work-unit frames (and later dynamic filters) flow leader -> worker through them. Empty + // when the embedder did not opt in: a ring latches `detached` once its sender count hits + // zero, so senders that might drop before every worker attached must never exist. + // The leader's `Cancel` senders are wired by the embedder (it shares the same outbound senders + // it holds for plan delivery and releases before the DSM unmaps), so drop the cancel set here. + let (outbound_senders, _cancel_senders) = + build_outbound_senders(0, n_procs, attach.outbound_senders); + Ok(LeaderAttach { + mesh: Arc::new(MppMesh::new(0, n_procs, inbound, interrupt, attach.alive)), + outbound_senders, + }) +} + +/// Build one task's work-unit feed channels, install the receiving ends on `cfg` (where the +/// deserialized plan's remote feed leaves look them up), and register the sending ends on +/// `mesh`'s drain so inbound `WorkUnit` frames fill them. `feeds` lists the task's declared +/// feeds as `(feed id, partitions)`, the same pairs the plan's `WorkUnitFeedDeclaration`s carry. +/// +/// The caller must keep the proc draining (a consumer loop, a send spin, or an explicit +/// [`crate::shm::CooperativeDrainSet::try_drain_pass`] pump) while a fragment waits on its +/// feed, or the units sit in the inbox unread. +/// Build the [`TaskMetrics`] payload for one executed fragment, for embedders that run +/// fragments outside the worker task registry (pg parallel workers). Pair it with +/// [`super::transport::MppSender::send_task_metrics_best_effort`] after the fragment's streams +/// complete; the leader-side rewrite consumes the same pre-order the in-registry path produces. +/// The task-level stamps (plan added/executed/finished) carry no data on this path, but the set +/// must be present: `decode_task_metrics` rejects a missing field, and one failed decode starves +/// the whole store. +/// +/// [`TaskMetrics`]: crate::proto::TaskMetrics +pub fn collect_task_metrics( + plan: &Arc, + task_index: usize, + task_count: usize, +) -> pb::TaskMetrics { + pb::TaskMetrics { + pre_order_plan_metrics: collect_plan_metrics_protos( + plan, + DistributedTaskContext { + task_index, + task_count, + }, + ), + task_metrics: Some(pb::MetricsSet::default()), + } +} + +pub fn install_work_unit_channels( + cfg: &mut datafusion::prelude::SessionConfig, + mesh: &MppMesh, + stage_id: u32, + task_number: u32, + feeds: &[(uuid::Uuid, usize)], +) { + let mut channels = RemoteWorkUnitFeedRegistry::default(); + for (id, partitions) in feeds { + channels.add(*id, *partitions); + } + cfg.set_extension(Arc::new(channels.receivers)); + mesh.register_work_unit_senders(stage_id, task_number, channels.senders); +} + +/// What [`worker_setup`] hands back to the embedder. +pub struct WorkerAttach { + /// The worker's mesh, installed on its DataFusion session. + pub mesh: Arc, + /// Outbound senders keyed by destination proc index. The slot at `this_proc` is the in-proc + /// self-loop; every other slot writes to that peer's inbox. + pub outbound_senders: Vec>, + /// The plan bytes the leader wrote into the region, copied out for this worker. + pub plan_bytes: Vec, +} + +/// Attach to the leader-initialized region as worker `proc_idx` (`>= 1`). +/// +/// # Safety +/// - `base`/`region_total` must match the region the leader initialized via [`leader_setup`]. +/// - `base` must be at least 8-byte (MAXALIGN) aligned; the ring headers hold atomics. +pub unsafe fn worker_setup( + base: *mut c_void, + region_total: usize, + proc_idx: u32, + wakeup: Arc, + receiver_token: u64, + interrupt: Arc, +) -> Result { + if receiver_token == NO_RECEIVER_TOKEN { + return Err(DataFusionError::Internal( + "mpp: worker_setup: receiver_token is the NO_RECEIVER_TOKEN sentinel; wakeups \ + for this proc would be silently disabled" + .into(), + )); + } + let (header, plan_bytes, attach) = + unsafe { worker_attach(base, region_total as u64, proc_idx, Arc::clone(&wakeup)) } + .map_err(DataFusionError::Internal)?; + let total_procs = header.n_procs; + + let (mut outbound, cancel) = + build_outbound_senders(proc_idx, total_procs, attach.outbound_senders); + + // Self-loop in-proc channel: peer-mesh routing can land a producer and its consumer on the same + // proc, and an MPSC inbox has no slot for a proc sending to itself. The unified drain pulls from + // both the inbox and this channel. + let (self_tx, self_rx) = in_proc_channel(SELF_LOOP_CAPACITY); + let self_tx_arc: Arc = Arc::new(self_tx); + outbound[proc_idx as usize] = Some(MppSender::with_header( + self_tx_arc, + MppFrameHeader::batch(0, 0, proc_idx), + )); + + let inbox = DsmInboxReceiver::new(attach.inbound_receiver); + inbox.set_receiver(receiver_token); + let inbound = Arc::new(DrainHandle::cooperative( + proc_idx, + vec![ + (ReceiverScope::Inbox, MppReceiver::new(Box::new(inbox))), + (ReceiverScope::SelfLoop, MppReceiver::new(Box::new(self_rx))), + ], + )); + let mesh = Arc::new(MppMesh::new( + proc_idx, + total_procs, + inbound, + interrupt, + attach.alive, + )); + // A worker consumes shuffle inputs, so it can be the consumer that stops a stream early. Give + // its mesh the control-plane cancel senders; they drop with the mesh at the end of the worker's + // run, well before the DSM unmaps, so no explicit release is needed. + mesh.set_cancel_senders(Arc::new(std::sync::Mutex::new(cancel))); + Ok(WorkerAttach { + mesh, + outbound_senders: outbound, + plan_bytes, + }) +} + +/// Run a producer fragment plan to exhaustion, pushing every output batch into the matching +/// per-partition [`PartitionSink`]. The output partition count of `plan` must equal `sinks.len()`; +/// `sinks[partition]` is the send end the caller routed for that output partition. +/// +/// Each partition's [`PartitionSink::finish`] sends a per-channel EOF when its stream ends, +/// regardless of how it ended: the shared queue is multiplexed across fragments, so dropping a sink +/// doesn't end the channel, only the EOF frame does. +pub async fn run_worker_fragment( + plan: Arc, + sinks: Vec>, + ctx: Arc, +) -> Result<()> { + let n_partitions = plan.output_partitioning().partition_count(); + if n_partitions != sinks.len() { + return Err(DataFusionError::Internal(format!( + "run_worker_fragment: plan has {n_partitions} output partitions but {} sinks", + sinks.len() + ))); + } + let mut futures = Vec::with_capacity(n_partitions); + for (partition, mut sink) in sinks.into_iter().enumerate() { + let plan = Arc::clone(&plan); + let ctx = Arc::clone(&ctx); + futures.push(async move { + let stream_result: Result<()> = async { + let mut stream = plan.execute(partition, ctx)?; + while let Some(batch) = stream.next().await { + let batch = batch?; + if batch.num_rows() == 0 { + continue; + } + sink.send(&batch).await?; + // Consumer abandoned this stream. Stop pulling: dropping `stream` ends the + // upstream scan and cascades the cancel to its own producers. + if sink.cancelled() { + break; + } + } + Ok(()) + } + .await; + let eof_result = sink.finish().await; + // Surface the stream error first, then any EOF-send error, so neither disappears. + stream_result.and(eof_result) + }); + } + // `join_all`, not `try_join_all`: fail-fast would cancel sibling partitions mid-await before + // they reach `finish`, leaving the consumer's channel buffer stuck. + let results = futures::future::join_all(futures).await; + for r in results { + r?; + } + Ok(()) +} diff --git a/src/shm/sink.rs b/src/shm/sink.rs new file mode 100644 index 000000000..a6ae19d7e --- /dev/null +++ b/src/shm/sink.rs @@ -0,0 +1,49 @@ +use async_trait::async_trait; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::Result; + +/// The producer's send end for one partition channel, symmetric to a [crate::WorkerChannel] read. +/// +/// This lives in the shm module rather than the core transport surface: only a push-based transport +/// (the shared-memory mesh) produces through sinks. Flight produces inside its gRPC worker service +/// and the in-memory transport pulls straight from the local task registry, so neither needs it. +/// +/// Contract with the produce loop: +/// - Batches arrive in `send` order and can be assumed non-empty. +/// - After a failed `send` the channel state is unspecified, but the caller still calls `finish` so +/// the consumer sees EOF; `finish` must tolerate a prior `send` error. +/// - Dropping a sink without calling `finish` does not end the channel, by design: `finish` is async +/// so Drop can't run it, and an implicit EOF would make an aborted producer look like a clean, +/// short stream. Abnormal teardown belongs to the transport, not the sink. +/// - `send` borrows the batch because transports serialize it into their own buffers; none needs +/// ownership. +#[async_trait] +pub trait PartitionSink: Send { + /// Sends one batch. Async so a blocked send can yield and let the transport make progress + /// elsewhere; a full channel must not park the calling thread. + async fn send(&mut self, batch: &RecordBatch) -> Result<()>; + /// Per-channel EOF, independent of the underlying link. Async for the same reason as `send`. + async fn finish(self: Box) -> Result<()>; + /// Whether the consumer cancelled this stream. The produce loop stops pulling its input when + /// this turns true, so a cancel doesn't just skip the send, it ends the upstream scan and drops + /// the input stream, cascading the cancel further up. Default `false` for links that don't carry + /// a cancel signal. + fn cancelled(&self) -> bool { + false + } +} + +/// The producer (write) side: opens a [PartitionSink] per output partition, symmetric to +/// [crate::WorkerChannel] (the read side). The worker's produce loop builds one and pushes each +/// output batch in. The shared-memory mesh provides the implementation; it is constructed by the +/// producer (which knows the per-partition routing), not handed out by the consume-side transport. +pub trait WorkerSink: Send + Sync { + /// Takes `stage` and `partition` separately because one sink serves every stage, unlike the + /// per-stage read connection that closes over its stage. + /// + /// `stage` is the producing stage's number and `partition` the producer task's own output + /// partition index, before routing. Several producer tasks of one stage may hold sinks for the + /// same pair, and the consumer merges them, so one `finish` is one producer task's EOF, not + /// channel completion (which stays transport-defined). + fn open_partition(&self, stage: usize, partition: usize) -> Result>; +} diff --git a/src/shm/transport.rs b/src/shm/transport.rs new file mode 100644 index 000000000..a9a60d99f --- /dev/null +++ b/src/shm/transport.rs @@ -0,0 +1,3539 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Transport layer for MPP shuffle. +//! +//! - [`MppFrameHeader`]: fixed 16-byte prefix tagging each wire message with +//! `(stage_id, partition)`, so one queue carries frames for many logical channels. +//! - [`encode_frame_into`] / [`decode_frame`]: Arrow IPC serialize/deserialize with +//! header prefix. Only codec entry points; tests round-trip through the same path. +//! - [`DrainBuffer`]: per-proc queue the drain writes into and the DataFusion consumer +//! reads from. Decouples consumer-side from producer-side backpressure: the drain +//! always makes forward progress on the inbound rings, so a stalled consumer can't +//! propagate backpressure to remote producers and cause a peer-mesh stall. + +use async_trait::async_trait; +use std::collections::VecDeque; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + +use datafusion::common::{HashMap, HashSet}; +use std::time::{Duration, Instant}; + +use datafusion::arrow::array::{ + ArrayRef, BinaryViewArray, RecordBatch, StringViewArray, UInt64Array, +}; +use datafusion::arrow::compute::take; +use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::ipc::reader::StreamReader; +use datafusion::arrow::ipc::writer::StreamWriter; +use datafusion::common::DataFusionError; +use prost::Message; + +use crate::common::deserialize_uuid; +use crate::proto as pb; +use crate::work_unit_feed::RemoteWorkUnitFeedTxs; +use crate::{WorkUnitMsg, set_received_time}; + +/// Magic bytes "MPPF" (MPP Frame) at the start of every wire message. +/// Lets receivers reject misrouted / corrupt frames before they hit Arrow IPC. +const MPP_FRAME_MAGIC: u32 = 0x4D505046; + +/// Wire-format size of [`MppFrameHeader`] in bytes. Asserted at compile time +/// below via `const _: ()`. +const MPP_FRAME_HEADER_SIZE: usize = 16; + +/// Kind of payload following [`MppFrameHeader`]. +/// +/// `Batch` is the common case. The header is followed by an Arrow IPC stream containing one +/// `RecordBatch`. `Eof` carries no payload. It signals the receiver that the named +/// `(stage_id, partition)` channel is done, even though the underlying shm_mq queue may still +/// carry frames for other channels. +/// +/// The remaining kinds are the control plane riding the same rings. For them the header's +/// `partition` field carries the task number instead: a work unit already names its +/// `(feed id, partition)` inside the payload, and metrics describe a whole task. +/// `SetPlan` (leader -> worker) carries one task's [`pb::SetPlanRequest`], the same message +/// Flight ships over its coordinator stream, plus the propagation headers that ride gRPC +/// metadata there; `WorkUnit` (leader -> worker) carries one prost-encoded work unit for +/// `(stage, task)`; `FeedEof` closes that task's feed channels (the wire stand-in for Flight's +/// stream close); `TaskMetrics` (worker -> leader) carries the task's collected metrics. +#[repr(u32)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum MppFrameKind { + Batch = 0, + Eof = 1, + WorkUnit = 2, + FeedEof = 3, + TaskMetrics = 4, + SetPlan = 5, + /// Consumer -> producer: stop producing the `(stage_id, partition)` stream, the consumer + /// stopped reading it before EOF (a top-N `LIMIT`, an inner merge join exhausting a side, etc.). + /// The producer ends that one stream cleanly. Scoped to the stream, not the connection, so the + /// ring stays healthy for metrics and every other stream, the way gRPC closes one stream + /// without dropping the channel. + Cancel = 6, + /// One piece of a frame too large for the channel's frame bound. The payload is a + /// `[total_len: u64][offset: u64]` prefix followed by that byte range of the inner frame. + /// The receiver reassembles per `(sender_proc, stage_id, partition)` and decodes the inner + /// frame once `total_len` bytes have arrived, so no single ring-resident message ever + /// exceeds the ring and frame size is never an error. + Chunk = 7, +} + +/// Payload of a `SetPlan` frame: the plan-delivery message a worker needs to run one task, +/// byte-compatible with what Flight sends. +/// +/// `set_plan` is the exact [`pb::SetPlanRequest`] the Flight dispatcher would put on its gRPC +/// stream. The headers are the config-extension propagation headers that travel as gRPC metadata +/// there; the ring has no metadata side channel, so they ride inside the frame as parallel +/// key/value lists (parallel lists rather than a map so repeated header names survive). +#[derive(Clone, PartialEq, prost::Message)] +pub struct SetPlanFrame { + #[prost(message, optional, tag = "1")] + pub set_plan: Option, + #[prost(string, repeated, tag = "2")] + pub header_keys: Vec, + #[prost(string, repeated, tag = "3")] + pub header_values: Vec, +} + +impl SetPlanFrame { + /// Bundle a plan-delivery message with the headers Flight would carry as gRPC metadata. + pub fn from_parts( + set_plan: pb::SetPlanRequest, + headers: &http::HeaderMap, + ) -> Result { + let mut header_keys = Vec::with_capacity(headers.len()); + let mut header_values = Vec::with_capacity(headers.len()); + for (name, value) in headers { + let value = value.to_str().map_err(|e| { + DataFusionError::Internal(format!( + "mpp: non-ASCII header {name} cannot travel in a SetPlan frame: {e}" + )) + })?; + header_keys.push(name.as_str().to_string()); + header_values.push(value.to_string()); + } + Ok(Self { + set_plan: Some(set_plan), + header_keys, + header_values, + }) + } + + /// Split back into the plan-delivery message and the propagation headers. + pub fn into_parts(self) -> Result<(pb::SetPlanRequest, http::HeaderMap), DataFusionError> { + let set_plan = self.set_plan.ok_or_else(|| { + DataFusionError::Internal("mpp: SetPlan frame carries no SetPlanRequest".to_string()) + })?; + let mut headers = http::HeaderMap::with_capacity(self.header_keys.len()); + for (key, value) in self.header_keys.iter().zip(self.header_values.iter()) { + let name = http::header::HeaderName::from_bytes(key.as_bytes()).map_err(|e| { + DataFusionError::Internal(format!("mpp: SetPlan frame header name {key:?}: {e}")) + })?; + let value = http::header::HeaderValue::from_str(value).map_err(|e| { + DataFusionError::Internal(format!("mpp: SetPlan frame header value for {key}: {e}")) + })?; + headers.append(name, value); + } + Ok((set_plan, headers)) + } +} + +/// 16-byte prefix on every transport frame. +/// +/// The fixed layout `[magic, flags, stage_id, partition]` (4×u32) is what +/// senders prepend before the Arrow IPC stream bytes and what receivers +/// parse before deciding which channel buffer the payload belongs to. +/// +/// See the `flags` bit-layout block below for the encoding of the `flags` word. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MppFrameHeader { + // Private so headers only come out of `batch()`/`eof()`: hand-built ones could bypass + // `pack_flags`'s sender bound and the reserved-bits invariant, and the consumer would + // reject them at decode, far from the producer. + pub(super) magic: u32, + pub(super) flags: u32, + pub(super) stage_id: u32, + pub(super) partition: u32, +} + +/// `flags` bit layout: +/// bits 0..8: frame kind (Batch | Eof) +/// bits 8..16: reserved (must be 0) +/// bits 16..32: sender_proc (mesh peer that wrote the frame) +const FRAME_KIND_MASK: u32 = 0x0000_00FF; +const FRAME_RESERVED_MASK: u32 = 0x0000_FF00; +const FRAME_SENDER_SHIFT: u32 = 16; +/// Maximum `sender_proc` representable in the header. Asserted at construction time so an +/// overflow becomes a hard error in the producer rather than silent flag corruption on the wire. +pub const MPP_MAX_SENDER_PROC: u32 = 0xFFFF; + +/// Spin bound for landing a control or metrics frame on a peer inbox that may be momentarily full. +/// A live peer drains its inbox within this many tries, so the frame lands well inside the bound; +/// it only runs out if the peer already exited, where the leader's wait-for-workers surfaces the +/// failure instead of hanging here. +const MAX_CONTROL_SEND_SPINS: usize = 10_000; + +const _: () = { + // shm_mq slot layout calculations depend on this being exact. + assert!(std::mem::size_of::() == MPP_FRAME_HEADER_SIZE); +}; + +#[inline] +fn pack_flags(kind: MppFrameKind, sender_proc: u32) -> u32 { + // fail_loud rather than debug_assert: in release builds the check would be compiled out and + // an out-of-range value would silently truncate to `sender_proc & 0xFFFF`. Catching the case + // where a refactor accidentally passes a task_id or partition here is the whole point. + assert!( + sender_proc <= MPP_MAX_SENDER_PROC, + "mpp: sender_proc {sender_proc} > MPP_MAX_SENDER_PROC ({MPP_MAX_SENDER_PROC})" + ); + (kind as u32) | (sender_proc << FRAME_SENDER_SHIFT) +} + +impl MppFrameHeader { + /// Build a `Batch` header for the given `(stage_id, partition)` stamped with `sender_proc`. + pub fn batch(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::Batch, sender_proc), + stage_id, + partition, + } + } + + /// Build an `Eof` header for the given `(stage_id, partition)` stamped with `sender_proc`. + /// Carries no payload; receivers route it to the channel buffer's source-done counter. + pub fn eof(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::Eof, sender_proc), + stage_id, + partition, + } + } + + /// Build a `WorkUnit` header addressed to `(stage_id, task_number)`. The `partition` slot + /// carries the task number; the unit's own `(feed id, partition)` ride in the payload. + pub fn work_unit(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::WorkUnit, sender_proc), + stage_id, + partition: task_number, + } + } + + /// Build a `FeedEof` header for `(stage_id, task_number)`: every feed of that task is done. + pub fn feed_eof(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::FeedEof, sender_proc), + stage_id, + partition: task_number, + } + } + + /// Build a `TaskMetrics` header for `(stage_id, task_number)`. + pub fn task_metrics(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::TaskMetrics, sender_proc), + stage_id, + partition: task_number, + } + } + + /// Build a `SetPlan` header for `(stage_id, task_number)`: the frame delivers that task's + /// plan to the proc hosting it. + pub fn set_plan(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::SetPlan, sender_proc), + stage_id, + partition: task_number, + } + } + + /// Build a `Cancel` header for the `(stage_id, partition)` stream, stamped with the consumer's + /// `sender_proc`. Carries no payload; the producer reads it as "stop sending this stream." + pub fn cancel(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::Cancel, sender_proc), + stage_id, + partition, + } + } + + /// Build a `Chunk` header for one piece of an oversized frame. `stage_id`/`partition` + /// mirror the inner frame's addressing: together with `sender_proc` they key the + /// receiver's reassembly, so chunked frames from two streams of the same proc can + /// interleave without corrupting each other. + pub fn chunk(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::Chunk, sender_proc), + stage_id, + partition, + } + } + + /// The mesh peer that wrote this frame. The drain demuxes incoming frames into the + /// per-channel buffer registry by `(sender_proc, stage_id, partition)`. + pub fn sender_proc(&self) -> u32 { + (self.flags >> FRAME_SENDER_SHIFT) & 0xFFFF + } + + /// Read the kind out of `flags`. Returns an error if the kind byte is + /// unknown or if any reserved bit (bits 8..16) is set, which catches wire-format + /// drift early. Sender_proc bits (16..32) are not validated here; readers extract + /// them with `sender_proc()`. + pub(super) fn kind(&self) -> Result { + let reserved = self.flags & FRAME_RESERVED_MASK; + if reserved != 0 { + return Err(DataFusionError::Internal(format!( + "mpp: reserved frame flag bits set ({reserved:#x})" + ))); + } + match self.flags & FRAME_KIND_MASK { + 0 => Ok(MppFrameKind::Batch), + 1 => Ok(MppFrameKind::Eof), + 2 => Ok(MppFrameKind::WorkUnit), + 3 => Ok(MppFrameKind::FeedEof), + 4 => Ok(MppFrameKind::TaskMetrics), + 5 => Ok(MppFrameKind::SetPlan), + 6 => Ok(MppFrameKind::Cancel), + 7 => Ok(MppFrameKind::Chunk), + other => Err(DataFusionError::Internal(format!( + "mpp: unknown frame kind {other:#x}" + ))), + } + } + + /// Serialize into the first `MPP_FRAME_HEADER_SIZE` bytes of `out`. + /// `out.len()` must be `>= MPP_FRAME_HEADER_SIZE`. + fn write_to(&self, out: &mut [u8]) { + debug_assert!(out.len() >= MPP_FRAME_HEADER_SIZE); + out[0..4].copy_from_slice(&self.magic.to_le_bytes()); + out[4..8].copy_from_slice(&self.flags.to_le_bytes()); + out[8..12].copy_from_slice(&self.stage_id.to_le_bytes()); + out[12..16].copy_from_slice(&self.partition.to_le_bytes()); + } + + /// Parse from the first `MPP_FRAME_HEADER_SIZE` bytes of `bytes`. Returns + /// `Err` if the slice is too short or the magic doesn't match. + fn parse(bytes: &[u8]) -> Result { + if bytes.len() < MPP_FRAME_HEADER_SIZE { + // No encoder in this file emits sub-header output, so a short frame means the + // shm_mq stitched together payloads from different senders. Hex-dump the bytes + // so the source is identifiable from log output without a debugger. + let hex = bytes + .iter() + .map(|b| format!("{b:02x}")) + .collect::>() + .join(" "); + return Err(DataFusionError::Internal(format!( + "mpp: frame too short for header ({} < {}); bytes = [{hex}]", + bytes.len(), + MPP_FRAME_HEADER_SIZE + ))); + } + let magic = u32::from_le_bytes(bytes[0..4].try_into().unwrap()); + if magic != MPP_FRAME_MAGIC { + return Err(DataFusionError::Internal(format!( + "mpp: bad frame magic {magic:#x} (expected {MPP_FRAME_MAGIC:#x})" + ))); + } + Ok(Self { + magic, + flags: u32::from_le_bytes(bytes[4..8].try_into().unwrap()), + stage_id: u32::from_le_bytes(bytes[8..12].try_into().unwrap()), + partition: u32::from_le_bytes(bytes[12..16].try_into().unwrap()), + }) + } +} + +/// Serialize `batch` into `buf` with a 16-byte [`MppFrameHeader`] prefix +/// addressing it to `(stage_id, partition)`. Wire format: +/// +/// ```text +/// [ magic | flags | stage_id | partition ] [ Arrow IPC stream bytes ] +/// |---------- 16 bytes --------| |---- variable ----| +/// ``` +/// +/// `flags` encodes kind + sender_proc; see the bit-layout block near +/// `FRAME_KIND_MASK` for details. +/// +/// Caller is expected to hold `buf` alive across many encodes so the peak-sized +/// allocation amortizes (~500 KB/batch on the 25M GROUP BY bench). +fn encode_frame_into( + header: MppFrameHeader, + batch: &RecordBatch, + buf: &mut Vec, +) -> Result<(), DataFusionError> { + buf.clear(); + buf.resize(MPP_FRAME_HEADER_SIZE, 0); + header.write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); + let mut writer = StreamWriter::try_new(&mut *buf, batch.schema_ref())?; + writer.write(batch)?; + writer.finish()?; + Ok(()) +} + +/// Copy `len` rows starting at `offset` into fresh, tightly-packed arrays. A plain +/// `RecordBatch::slice` keeps the original backing buffers, and arrow-ipc writes a sliced +/// variable-length array's values buffer whole, which is exactly the ballooning the oversized +/// send path exists to undo. `take` materializes only the selected rows; view arrays +/// additionally need a `gc`, because `take` copies their views but keeps referencing the +/// shared data blocks, so the encoded size wouldn't shrink no matter how few rows remain. +/// +/// The flight path applies the same collection idea in `garbage_collect_arrays` +/// (`protocol/grpc/worker_service.rs`), unconditionally and with dictionary gc on top. +/// They stay separate on purpose: everything outside `shm` follows upstream, and the +/// dictionary kernel lives in `arrow-select`, which only the `grpc` feature pulls in. +/// Dictionaries therefore ride uncollected here: `take` re-maps the keys but keeps the +/// whole values array, so an oversized dictionary frame streams in chunks instead of +/// shrinking. +fn compact_rows( + batch: &RecordBatch, + offset: usize, + len: usize, +) -> Result { + let indices = UInt64Array::from_iter_values(offset as u64..(offset + len) as u64); + let columns = batch + .columns() + .iter() + .map(|column| { + let taken = take(column.as_ref(), &indices, None)?; + Ok(match taken.data_type() { + DataType::Utf8View => Arc::new( + taken + .as_any() + .downcast_ref::() + .unwrap() + .gc(), + ) as ArrayRef, + DataType::BinaryView => Arc::new( + taken + .as_any() + .downcast_ref::() + .unwrap() + .gc(), + ) as ArrayRef, + _ => taken, + }) + }) + .collect::, DataFusionError>>()?; + Ok(RecordBatch::try_new(batch.schema(), columns)?) +} + +/// Serialize a payload-less [`MppFrameKind::Eof`] frame for `(stage_id, partition)` +/// into `buf`. The shm_mq peer reads this as a 16-byte message and routes it to +/// the channel buffer's source-done counter without touching Arrow IPC. +/// Consumed by [`MppSender::send_eof_traced`] when a producer fragment's +/// per-partition stream exhausts, so the receiver's `(stage_id, partition)` +/// channel buffer transitions to `Eof` even though the multiplexed shm_mq queue +/// stays attached for other channels. +fn encode_eof_frame_into( + stage_id: u32, + partition: u32, + sender_proc: u32, + buf: &mut Vec, +) -> Result<(), DataFusionError> { + buf.clear(); + buf.resize(MPP_FRAME_HEADER_SIZE, 0); + MppFrameHeader::eof(stage_id, partition, sender_proc) + .write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); + Ok(()) +} + +/// Byte length of the `[total_len: u64][offset: u64]` prefix on every `Chunk` payload. +const CHUNK_PREFIX_SIZE: usize = 16; + +/// Serialize one piece of an oversized frame: chunk header, `[total_len, offset]` prefix, then +/// `data` (the inner frame's bytes at `offset`). +fn encode_chunk_frame_into( + header: MppFrameHeader, + total_len: u64, + offset: u64, + data: &[u8], + buf: &mut Vec, +) { + buf.clear(); + buf.resize(MPP_FRAME_HEADER_SIZE, 0); + header.write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); + buf.extend_from_slice(&total_len.to_le_bytes()); + buf.extend_from_slice(&offset.to_le_bytes()); + buf.extend_from_slice(data); +} + +/// Parse a `Chunk` frame's payload into `(total_len, offset, data)`. The caller has already +/// parsed and validated the 16-byte frame header. +fn parse_chunk_frame(bytes: &[u8]) -> Result<(usize, usize, &[u8]), DataFusionError> { + let payload = &bytes[MPP_FRAME_HEADER_SIZE..]; + if payload.len() < CHUNK_PREFIX_SIZE { + return Err(DataFusionError::Internal(format!( + "mpp: chunk frame too short for its prefix ({} < {CHUNK_PREFIX_SIZE})", + payload.len() + ))); + } + let total_len = u64::from_le_bytes(payload[0..8].try_into().unwrap()); + let offset = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + let (Ok(total_len), Ok(offset)) = (usize::try_from(total_len), usize::try_from(offset)) else { + return Err(DataFusionError::Internal(format!( + "mpp: chunk prefix out of range (total_len {total_len}, offset {offset})" + ))); + }; + Ok((total_len, offset, &payload[CHUNK_PREFIX_SIZE..])) +} + +/// Serialize a prost-encoded control payload (`WorkUnit` / `TaskMetrics`) behind `header`. +fn encode_prost_frame_into( + header: MppFrameHeader, + msg: &impl prost::Message, + buf: &mut Vec, +) -> Result<(), DataFusionError> { + buf.clear(); + buf.resize(MPP_FRAME_HEADER_SIZE, 0); + header.write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); + msg.encode(buf) + .map_err(|e| DataFusionError::Internal(format!("mpp: prost frame encode: {e}")))?; + Ok(()) +} + +/// A decoded frame payload, routed by the drain according to its kind. +#[derive(Debug)] +enum FrameBody { + Batch(RecordBatch), + Eof, + WorkUnit(pb::WorkUnit), + FeedEof, + TaskMetrics(pb::TaskMetrics), + SetPlan(SetPlanFrame), + Cancel, +} + +/// Inverse of the frame encoders. Parses the 16-byte header and decodes the payload according +/// to the kind. Receivers branch on the body to decide routing. +fn decode_frame(bytes: &[u8]) -> Result<(MppFrameHeader, FrameBody), DataFusionError> { + let header = MppFrameHeader::parse(bytes)?; + let payload = &bytes[MPP_FRAME_HEADER_SIZE..]; + match header.kind()? { + MppFrameKind::Eof | MppFrameKind::FeedEof | MppFrameKind::Cancel => { + if bytes.len() != MPP_FRAME_HEADER_SIZE { + return Err(DataFusionError::Internal(format!( + "mpp: payload-less frame carries payload ({} > {})", + bytes.len(), + MPP_FRAME_HEADER_SIZE + ))); + } + match header.kind()? { + MppFrameKind::Eof => Ok((header, FrameBody::Eof)), + MppFrameKind::Cancel => Ok((header, FrameBody::Cancel)), + _ => Ok((header, FrameBody::FeedEof)), + } + } + MppFrameKind::WorkUnit => { + let unit = pb::WorkUnit::decode(payload) + .map_err(|e| DataFusionError::Internal(format!("mpp: work unit decode: {e}")))?; + Ok((header, FrameBody::WorkUnit(unit))) + } + MppFrameKind::TaskMetrics => { + let metrics = pb::TaskMetrics::decode(payload) + .map_err(|e| DataFusionError::Internal(format!("mpp: task metrics decode: {e}")))?; + Ok((header, FrameBody::TaskMetrics(metrics))) + } + MppFrameKind::SetPlan => { + let frame = SetPlanFrame::decode(payload) + .map_err(|e| DataFusionError::Internal(format!("mpp: set-plan decode: {e}")))?; + Ok((header, FrameBody::SetPlan(frame))) + } + MppFrameKind::Batch => { + let mut reader = StreamReader::try_new(payload, None)?; + let batch = reader.next().ok_or_else(|| { + DataFusionError::Execution("mpp: empty arrow-ipc stream in decode_frame".into()) + })??; + Ok((header, FrameBody::Batch(batch))) + } + MppFrameKind::Chunk => Err(DataFusionError::Internal( + "mpp: chunk frame reached decode_frame; chunks reassemble in MppReceiver first" + .to_string(), + )), + } +} + +/// Local queue between a drain (either the cooperative `try_drain_pass` or the test-only thread +/// variant) and the consumer that pops batches. +/// +/// In the cooperative path each `DrainBuffer` corresponds to one logical channel: one +/// `(stage_id, partition)` entry in the owning [`DrainHandle`]'s registry. `num_sources` is +/// always `1` there because a given drain serves a single sender_proc, which is the only producer +/// for any channel routed through it. The test-only thread path uses a single shared buffer with +/// `num_sources = N` over an N-sender setup. +/// +/// Push side: callers append deserialized batches; on source detach (or per-channel `Eof` frame) +/// [`DrainBuffer::notify_source_done`] is called. Once `sources_done >= num_sources` AND the +/// queue is empty, `try_pop` returns [`DrainItem::Eof`]. +/// +/// Pop side: cooperative consumers loop on `try_pop` + `yield_now`. The test-only `pop_front` +/// blocks on the condvar. +#[derive(Debug)] +pub(super) struct DrainBuffer { + inner: Mutex, + cond: Condvar, +} + +#[derive(Debug)] +struct DrainBufferInner { + queue: VecDeque, + num_sources: u32, + sources_done: u32, + /// Consumer-side cancel flag. When set (e.g., query cancelled or `DrainHandle` dropped), + /// `try_pop`/`pop_front` returns `Eof` even if `sources_done` hasn't reached `num_sources`. + cancelled: bool, + /// Set when the receiver feeding this channel detached (or errored) before the channel's + /// `Eof` frame arrived. Distinct from `cancelled`: cancellation is a clean teardown and + /// yields `Eof`, while a lost source must surface as an error or the consumer would treat + /// truncated output as complete. + failed: Option, +} + +/// Yielded by [`DrainBuffer::pop_front`]. +#[derive(Debug)] +pub(super) enum DrainItem { + /// A batch produced by one of the inbound shm_mqs. + Batch(RecordBatch), + /// All source queues have detached and the local queue is drained. + Eof, + /// The receiver feeding this channel went away before the channel's `Eof` frame. + Failed(String), +} + +impl DrainBuffer { + /// Create a drain buffer expecting `num_sources` inbound queues. For a + /// proc in an N-proc mesh, `num_sources == N - 1` (all peers + /// excluding self — the self-partition bypasses the buffer). + pub fn new(num_sources: u32) -> Arc { + Arc::new(Self { + inner: Mutex::new(DrainBufferInner { + queue: VecDeque::new(), + num_sources, + sources_done: 0, + cancelled: false, + failed: None, + }), + cond: Condvar::new(), + }) + } + + /// Push a freshly-received batch into the local queue. + pub fn push_batch(&self, batch: RecordBatch) { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + guard.queue.push_back(batch); + self.cond.notify_one(); + } + + /// Mark one source queue as detached. Safe to call from the drain thread + /// after observing `SHM_MQ_DETACHED` on a given inbound queue. + pub fn notify_source_done(&self) { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + guard.sources_done = guard.sources_done.saturating_add(1); + if guard.sources_done >= guard.num_sources { + self.cond.notify_all(); + } + } + + /// Mark the channel as fed by a dead receiver, unless it already completed (its `Eof` + /// arrived), was cancelled, or already failed. Consumers then see an error instead of + /// hanging on a channel nothing will ever fill. + pub fn fail_pending(&self, msg: &str) { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + if guard.sources_done >= guard.num_sources || guard.cancelled || guard.failed.is_some() { + return; + } + guard.failed = Some(msg.to_string()); + self.cond.notify_all(); + } + + /// Cancel all further pushes and wake all consumers with EOF. + pub fn cancel(&self) { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + guard.cancelled = true; + self.cond.notify_all(); + } + + /// Non-blocking variant. Returns the front item, or `DrainItem::Eof` if + /// all sources have detached and the queue is drained, or `None` if more + /// data may yet arrive. Cooperative consumers loop on + /// `try_drain_pass` + `try_pop`, yielding to the executor between + /// iterations. + pub fn try_pop(&self) -> Option { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + Self::try_pop_locked(&mut guard) + } + + /// Shared body of [`try_pop`] and the test-only [`Self::pop_front`]. + /// Returns `Some(Batch)` if the queue has data, `Some(Eof)` if all + /// sources have detached or the buffer is cancelled, and `None` + /// otherwise. Lets the two entry points stay in lockstep on the + /// "buffered data wins over cancellation/EOF" invariant locked in by + /// the `drain_buffer_drains_buffered_before_eof` test. + fn try_pop_locked(guard: &mut MutexGuard<'_, DrainBufferInner>) -> Option { + if let Some(batch) = guard.queue.pop_front() { + return Some(DrainItem::Batch(batch)); + } + if let Some(msg) = &guard.failed { + return Some(DrainItem::Failed(msg.clone())); + } + if guard.cancelled || guard.sources_done >= guard.num_sources { + return Some(DrainItem::Eof); + } + None + } +} + +/// Outcome of a single non-blocking receive attempt. +#[derive(Debug)] +pub(super) enum RecvOutcome { + /// One serialized Arrow IPC message ready to decode. + Bytes(Vec), + /// No data currently available but the peer is still attached. + Empty, + /// The peer has detached; no more bytes will ever arrive on this channel. + Detached, +} + +/// Non-blocking byte channel receiver. Implementations: `DsmInboxReceiver` (production), +/// `std::sync::mpsc` (tests). Must be `Send` because the drain thread takes ownership. +pub(super) trait BatchChannelReceiver: Send + Sync { + fn try_recv(&self) -> RecvOutcome; +} + +/// Byte channel sender paired with [`BatchChannelReceiver`]. `send` blocks when +/// the channel is full. Dropping the sender signals EOF to the receiver. +/// +/// `Send` is required because unit tests and future producer-pump threads move +/// senders across thread boundaries. +pub(crate) trait BatchChannelSender: Send + Sync { + fn send_bytes(&self, bytes: &[u8]) -> Result<(), DataFusionError>; + + /// Non-blocking variant. Returns `Ok(true)` on success, `Ok(false)` + /// when the channel is full (caller should retry), `Err` on detach / + /// transport error. Default falls back to the blocking send — safe + /// for in-proc channels used by tests where "full" doesn't arise. + fn try_send_bytes(&self, bytes: &[u8]) -> Result { + self.send_bytes(bytes).map(|()| true) + } + + /// Async lock the send paths hold across the cooperative-drain spin so two tasks can't + /// interleave partial writes on the same handle. PG's `shm_mq_send` requires the same + /// `(nbytes, data)` on retry after `SHM_MQ_WOULD_BLOCK`. Multiple [`MppSender`] clones + /// multiplex onto one channel, and the spin's `yield_now().await` would otherwise let a + /// sibling task land a different payload mid-message and corrupt the queue. In-proc + /// channels return a per-instance mutex too, just to keep the call sites uniform. + fn send_lock(&self) -> &tokio::sync::Mutex<()>; + + /// Largest single frame the channel accepts, when bounded. The batch send path splits + /// oversized batches to fit under it. `None` (the default, for in-proc channels) disables + /// splitting. + fn max_frame_bytes(&self) -> Option { + None + } +} + +/// Pluggable "drain everything inbound" hook for [`MppSender`]'s cooperative send spin. The +/// peer-mesh deadlock-breaking pattern needs the producer to pump ALL inbound queues (not just +/// one) while waiting for a full outbound, so the implementation typically delegates to +/// `MppMesh::drain_all_inbound()` which iterates every per-sender-proc drain. +pub trait CooperativeDrainSet: Send + Sync { + fn try_drain_pass(&self) -> Result<(), DataFusionError>; + + /// Checked in the send spin alongside `try_drain_pass`: returns `Err` (or aborts) if the + /// query should stop. Default is a no-op, for embedders with no external interrupt source; a + /// Postgres embedder overrides this to run `check_for_interrupts!`, which longjmps on cancel. + fn check_interrupt(&self) -> Result<(), DataFusionError> { + Ok(()) + } + + /// Whether a consumer cancelled the `(stage_id, partition)` stream (a `Cancel` frame arrived on + /// this proc's inbox). The send spin ends the producer's stream cleanly when it's set. Default + /// `false` for drains that don't carry inbound control frames (in-proc test channels). + fn stream_cancelled(&self, _stage_id: u32, _partition: u32) -> bool { + false + } +} + +/// Cancellation extension point, checked at the transport's block points (the send spin and the consumer +/// pull loop). An in-process embedder uses the default no-op or a cancellation token; a Postgres +/// embedder runs `check_for_interrupts!`, which longjmps out of the backend on cancel. +pub trait Interrupt: Send + Sync { + fn check(&self) -> Result<(), DataFusionError>; +} + +/// No-op interrupt for embedders that have no external cancellation source. +pub struct NoInterrupt; +impl Interrupt for NoInterrupt { + fn check(&self) -> Result<(), DataFusionError> { + Ok(()) + } +} + +impl CooperativeDrainSet for DrainHandle { + fn try_drain_pass(&self) -> Result<(), DataFusionError> { + DrainHandle::try_drain_pass(self) + } + + fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { + DrainHandle::stream_cancelled(self, stage_id, partition) + } +} + +/// High-level sender: encodes a `RecordBatch` then pushes bytes through the underlying channel. +/// +/// With `cooperative_drain` set, `send_batch` breaks the symmetric-send deadlock on a +/// single-threaded tokio runtime by interleaving send-retries with +/// `CooperativeDrainSet::try_drain_pass` on the same mesh's inbound side. Each proc's +/// sender doing the same guarantees mutual progress: our drain pulls peer-shipped rows out of +/// our inbound queues, which frees peers' outbound-to-us send space, which lets their sends +/// un-stall. +pub struct MppSender { + /// Underlying byte channel. Held behind `Arc` so multiple `MppSender`s can share one + /// `shm_mq` queue while tagging frames with different `(stage_id, partition)` headers, which + /// is the multiplexed path's natural pattern. Clone the Arc, build a new `MppSender` with a + /// different header, both write into the same queue. + pub(super) channel: Arc, + cooperative_drain: Option>, + /// Frame header prepended to every outgoing batch. Identifies the logical + /// `(stage_id, partition)` channel the receiver demultiplexes on. Per-sender rather than + /// per-call: each partition gets its own `MppSender` via `clone_with_header`, all sharing + /// the underlying `Arc` of a single shm_mq queue. + pub(super) header: MppFrameHeader, + /// Scratch buffer reused across every `encode_frame_into` on this sender. Sized by the + /// first batch; subsequent batches clear and re-fill without reallocating. Interior + /// mutability lets the caller keep the `&self` signature (each producer fragment holds + /// its `MppSender` clones behind shared borrows for the duration of + /// `worker::run_worker_fragment`). + scratch: std::cell::RefCell>, +} + +// SAFETY: only `scratch: RefCell>` and the trait-object `Arc`s are `!Sync`. Callers +// compose `send_*_traced` futures via `tokio::spawn` / `join_all`, which makes the compiler +// require `&Self: Send` and therefore `Self: Sync`. The shared-memory model runs those futures on +// a current-thread runtime (see the module docs), so the cell is never observed from two +// threads; a multi-thread embedder would additionally be serialized by `send_lock` across +// every send path that touches `scratch`. +unsafe impl Sync for MppSender {} + +impl MppSender { + /// Construct a sender that tags every outgoing batch with `header`. Production call sites + /// clone one shared `Arc` across N senders, each with a different + /// `MppFrameHeader::batch(stage, p)`. That's the multiplexed pattern for fanning multiple + /// partitions over one shm_mq queue. + pub(super) fn with_header( + channel: Arc, + header: MppFrameHeader, + ) -> Self { + Self { + channel, + cooperative_drain: None, + header, + scratch: std::cell::RefCell::new(Vec::new()), + } + } + + /// Build a new `MppSender` that shares this sender's underlying channel + /// but tags every frame with `header` instead. Used by callers that know + /// the physical plan's output partition count and need one sender per + /// partition, all multiplexed over the same shm_mq queue. + pub fn clone_with_header(&self, header: MppFrameHeader) -> Self { + Self { + channel: Arc::clone(&self.channel), + cooperative_drain: self.cooperative_drain.as_ref().map(Arc::clone), + header, + scratch: std::cell::RefCell::new(Vec::new()), + } + } + + /// Whether the consumer of this sender's `(stage, partition)` stream cancelled it. Read by the + /// produce loop to stop pulling its input, not just skip the send. `false` without a drain + /// (in-proc test channels carry no inbound cancel). + pub(super) fn stream_cancelled(&self) -> bool { + self.cooperative_drain + .as_ref() + .is_some_and(|d| d.stream_cancelled(self.header.stage_id, self.header.partition)) + } + + /// Attach a [`CooperativeDrainSet`] so `Self::send_batch_traced`'s spin + /// can drain inbound peer traffic while waiting for outbound space. + /// Required for peer-mesh fragments where every worker is both sender and + /// consumer; without it, symmetric full-queue stalls deadlock the + /// single-threaded Tokio runtime. + pub fn with_cooperative_drain(mut self, drain: Arc) -> Self { + self.cooperative_drain = Some(drain); + self + } + + /// `send_batch` variant that accumulates per-call timings and spin counts into `stats`. + /// Callers that report at EOF (e.g. `ShuffleStream`) use this to diagnose where time + /// goes when the outbound queue is full. + /// + /// Async because the spin awaits the per-handle send lock and yields between + /// `try_send_bytes` retries; see `send_with_scratch`. + pub(super) async fn send_batch_traced( + &self, + batch: &RecordBatch, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + // Take the scratch buffer out of the `RefCell` rather than + // holding a `RefMut` across the spin below. The spin contains + // the embedder's `Interrupt::check`, which may unwind or `longjmp` through + // Rust frames; a `longjmp` does not run `Drop`, so a `RefMut` + // held across it would leave the cell perpetually borrowed and + // panic the next caller. `replace` is atomic — the cell is + // never observed in a borrowed state — and we put the buffer + // back at the end so its heap allocation survives across calls. + // If the spin longjmps anyway, the cell holds the default empty + // `Vec` and the next call simply re-allocates. + let mut scratch = self.scratch.replace(Vec::new()); + let result = self.send_with_scratch(batch, &mut scratch, stats).await; + self.scratch.replace(scratch); + result + } + + /// Send a payload-less [`MppFrameKind::Eof`] frame so the receiver's `(stage_id, partition)` + /// channel buffer transitions to `Eof` and the consumer's pull loop terminates cleanly. + /// + /// Producer fragments must call this exactly once per `(stage_id, partition)` channel after + /// the local stream exhausts. Without it the multiplexed shm_mq queue stays attached (other + /// channels still flow) and the consumer channel buffer never reaches `sources_done == 1`. The + /// receive-side [`DrainHandle::try_drain_pass`] decodes the frame and calls + /// `notify_source_done` on the matching channel buffer. + /// + /// Uses the same cooperative-spin path as [`Self::send_batch_traced`] so a full outbound + /// queue doesn't deadlock the EOF send. `stats.spin_iters` / `send_wait` capture any + /// contention. + /// + /// Symmetric-EOF safety: when every peer reaches EOF simultaneously with full outbound + /// queues, each peer's cooperative [`CooperativeDrainSet::try_drain_pass`] inside the spin + /// pulls peer-sent frames out of its own inbound queues, freeing space the peers are blocked + /// on. Progress is monotone: at least one `try_send_bytes` succeeds per spin iteration + /// somewhere in the mesh, so symmetric stalls resolve within a few iterations rather than + /// deadlocking. + pub(super) async fn send_eof_traced( + &self, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = self.send_eof_with_scratch(&mut scratch, stats).await; + self.scratch.replace(scratch); + result + } + + /// Bounded synchronous send of a `Cancel` frame for the `(stage_id, partition)` stream. The + /// consumer calls it on the sender to the producing proc when it abandons that stream. + /// Synchronous so it can run from the consumer stream's drop, where `await` isn't available. + /// + /// The bound doesn't risk a stuck producer. A live producer drains its own inbox in its send + /// spin, so a slot frees and the frame lands well inside the bound even when the inbox is + /// backed up. The bound only runs out if the producer already exited or died, and a dead worker + /// makes the leader's wait-for-workers error out rather than hang. + pub fn try_send_cancel(&self, stage_id: u32, partition: u32) { + let header = MppFrameHeader::cancel(stage_id, partition, self.header.sender_proc()); + let mut buf = [0u8; MPP_FRAME_HEADER_SIZE]; + header.write_to(&mut buf); + for _ in 0..MAX_CONTROL_SEND_SPINS { + match self.channel.try_send_bytes(&buf) { + Ok(true) => return, + Ok(false) => std::thread::yield_now(), + Err(_) => return, // the producer's inbox is gone; nothing left to cancel + } + } + } + + async fn send_eof_with_scratch( + &self, + scratch: &mut Vec, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + encode_eof_frame_into( + self.header.stage_id, + self.header.partition, + self.header.sender_proc(), + scratch, + )?; + self.spin_send_frame(scratch, stats).await.map(|_| ()) + } + + /// Spin-loop helper: call `channel.try_send_bytes(scratch)`. + async fn spin_try_send_bytes(&self, scratch: &[u8]) -> Result { + self.channel.try_send_bytes(scratch) + } + + /// Spin-loop helper: call `drain.try_drain_pass()`. + async fn spin_try_drain_pass( + &self, + drain: &Arc, + ) -> Result<(), DataFusionError> { + drain.try_drain_pass() + } + + async fn send_with_scratch( + &self, + batch: &RecordBatch, + scratch: &mut Vec, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let t_enc = Instant::now(); + encode_frame_into(self.header, batch, scratch)?; + stats.encode += t_enc.elapsed(); + if self + .channel + .max_frame_bytes() + .is_none_or(|cap| scratch.len() <= cap) + { + return self.spin_send_scratch(scratch, stats).await; + } + // An over-cap frame usually means the batch carries offset slices of an upstream + // operator's accumulated state: arrow-ipc writes a sliced variable-length array's + // whole values buffer, so the frame balloons to the state's size no matter the row + // count. Compacting into fresh arrays drops the shared buffers. It's an + // optimization, not a bound: whatever stays over the cap streams through the ring + // in chunks. + let compacted = compact_rows(batch, 0, batch.num_rows())?; + let t_enc = Instant::now(); + encode_frame_into(self.header, &compacted, scratch)?; + stats.encode += t_enc.elapsed(); + self.spin_send_scratch(scratch, stats).await + } + + /// Push an already-encoded frame through the channel, splitting it into `Chunk` frames + /// first when it exceeds the channel's frame bound. Shared by every frame kind's send + /// path, so an oversized `SetPlan` streams the same way an oversized batch does. + async fn spin_send_scratch( + &self, + scratch: &[u8], + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + match self.channel.max_frame_bytes() { + Some(cap) if scratch.len() > cap => self.send_chunked(scratch, cap, stats).await, + _ => self.spin_send_frame(scratch, stats).await.map(|_| ()), + } + } + + /// Stream one oversized frame through the bounded channel as a sequence of `Chunk` + /// frames, each within the channel's bound. The whole frame exists only in this + /// sender's scratch and the receiver's reassembly buffer, never in the ring, so the + /// ring bounds backpressure granularity rather than message size. A cancel observed + /// between chunks abandons the remainder; the receiver's reassembly rules (offset-0 + /// restart, cleanup on the stream's `Eof`) tolerate the orphaned prefix. + async fn send_chunked( + &self, + frame: &[u8], + cap: usize, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let Some(chunk_data_max) = cap + .checked_sub(MPP_FRAME_HEADER_SIZE + CHUNK_PREFIX_SIZE) + .filter(|n| *n > 0) + else { + return Err(DataFusionError::Internal(format!( + "mpp: channel frame bound {cap} cannot fit a chunk header" + ))); + }; + let header = MppFrameHeader::chunk( + self.header.stage_id, + self.header.partition, + self.header.sender_proc(), + ); + let total_len = frame.len() as u64; + let mut buf = + Vec::with_capacity(MPP_FRAME_HEADER_SIZE + CHUNK_PREFIX_SIZE + chunk_data_max); + let mut offset = 0usize; + while offset < frame.len() { + let end = usize::min(offset + chunk_data_max, frame.len()); + encode_chunk_frame_into( + header, + total_len, + offset as u64, + &frame[offset..end], + &mut buf, + ); + if !self.spin_send_frame(&buf, stats).await? { + // The consumer cancelled the stream mid-frame; drop the rest. + return Ok(()); + } + offset = end; + } + Ok(()) + } + + /// Push one channel-bounded frame through the cooperative-drain spin (or the blocking + /// fallback when no drain is attached). Returns `false` when the spin ended because the + /// consumer cancelled this sender's stream, so multi-frame callers can stop early. + async fn spin_send_frame( + &self, + scratch: &[u8], + stats: &mut SendBatchStats, + ) -> Result { + let Some(drain) = self.cooperative_drain.as_ref() else { + // No drain attached (unit tests, in-proc channels): fall + // back to the blocking send path. + return self.channel.send_bytes(scratch).map(|()| true); + }; + // Lock the channel before the spin so a sibling task can't interleave a different + // partial write through the shared shm_mq handle. See `BatchChannelSender::send_lock`. + // Long-term, switching shm_mq for an async-friendly ring buffer (cf. #4184) drops the + // partial-send invariant entirely and removes the need for this lock. + // + // Latent under the current-thread runtime: today every fragment owns its own + // `Arc` (one sender per `Arc`), so the FIFO Mutex below + // is uncontended. A future multi-thread runtime that shares a sender across + // sibling fragment tasks (multi-partition fan-out) would let one task starve + // another for the duration of a large shuffle; the fix at that point is to move + // the entire spin off the compute thread. + let _send_guard = self.channel.send_lock().lock().await; + let mut first_try = true; + let t_wait_start = Instant::now(); + // The spin runs inside a tokio task on the backend thread's current-thread runtime + // (DataFusion needs one to drive `Stream`s). The deadlock we're breaking is + // *cross-proc*: two peers each blocked on a full outbound. `try_drain_pass` pulls + // peer batches off our inbound on the same OS thread, freeing their slots so their + // sends advance. `yield_now().await` between iterations hands the runtime back to + // siblings if any are ready, mostly a no-op under today's linear MPP topology. + loop { + drain.check_interrupt()?; + // The consumer cancelled this stream, so end the send cleanly and let the producer + // fragment complete. Checked before the send so a cancel that landed mid-spin stops + // the next iteration. + if drain.stream_cancelled(self.header.stage_id, self.header.partition) { + return Ok(false); + } + if self.spin_try_send_bytes(scratch).await? { + if !first_try { + stats.send_wait += t_wait_start.elapsed(); + } + return Ok(true); + } + first_try = false; + stats.spin_iters += 1; + // Would-block. Pull from our inbound so peers' outbound-to-us frees up and their + // sends to us unblock; without this, symmetric full-queue sends deadlock. Errors + // propagate so a peer detaching mid-spin doesn't leave us spinning on a closed + // mesh. + let t_drain = Instant::now(); + self.spin_try_drain_pass(drain).await?; + stats.coop_drain_in_spin += t_drain.elapsed(); + tokio::task::yield_now().await; + } + } + + /// Send one work unit for the task this sender's header names. The unit's hop stamps are the + /// caller's job; the payload travels prost-encoded, never through Arrow IPC. + pub async fn send_work_unit_traced( + &self, + unit: &pb::WorkUnit, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + encode_prost_frame_into(self.header, unit, &mut scratch)?; + self.spin_send_scratch(&scratch, stats).await + } + .await; + self.scratch.replace(scratch); + result + } + + /// Ship one task's plan as a `SetPlan` frame: the wire stand-in for Flight's + /// `SetPlanRequest` over its coordinator stream. + pub async fn send_set_plan_traced( + &self, + frame: &SetPlanFrame, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + encode_prost_frame_into(self.header, frame, &mut scratch)?; + self.spin_send_scratch(&scratch, stats).await + } + .await; + self.scratch.replace(scratch); + result + } + + /// Close the feed channels of the task this sender's header names: the wire stand-in for + /// Flight closing its coordinator stream, after which the worker-side feed streams end. + pub async fn send_feed_eof_traced( + &self, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + scratch.clear(); + scratch.resize(MPP_FRAME_HEADER_SIZE, 0); + MppFrameHeader::feed_eof( + self.header.stage_id, + self.header.partition, + self.header.sender_proc(), + ) + .write_to(&mut scratch[..MPP_FRAME_HEADER_SIZE]); + self.spin_send_scratch(&scratch, stats).await + } + .await; + self.scratch.replace(scratch); + result + } + + /// Send the task's collected metrics without consulting the interrupt: this runs after the + /// query's cancellation token already fired (it fires on normal completion too), so the + /// cooperative spin would abort exactly when delivery matters. Best-effort like Flight's + /// metrics sends: a detached leader drops them. Retrying on a full ring is safe because the + /// receiving side keeps draining until every producer reported in. + pub async fn send_task_metrics_best_effort( + &self, + metrics: &pb::TaskMetrics, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + encode_prost_frame_into(self.header, metrics, &mut scratch)?; + let _send_guard = self.channel.send_lock().lock().await; + // The consumer stops reading the data path early but keeps draining at teardown to + // collect these metrics, so the frame lands once the data the cancel stopped drains out + // and a slot frees. Bounded so a leader that really went away can't wedge the worker on + // a full ring. + for _ in 0..MAX_CONTROL_SEND_SPINS { + match self.channel.try_send_bytes(&scratch) { + Ok(true) => return Ok(()), + Ok(false) => tokio::task::yield_now().await, + Err(_) => return Ok(()), // receiver gone; metrics are best-effort + } + } + Ok(()) + } + .await; + self.scratch.replace(scratch); + result + } +} + +/// Per-call timing + spin metrics for [`MppSender::send_batch_traced`]. +/// All fields accumulate; callers zero or reuse as needed. +#[derive(Default, Debug, Clone)] +pub struct SendBatchStats { + /// Cumulative time spent inside `encode_frame_into` (header + Arrow IPC serialization). + pub encode: Duration, + /// Cumulative wall time in the send-retry spin after the first failed + /// `try_send_bytes`. Zero if the first try succeeded. + pub send_wait: Duration, + /// Cumulative time spent in `try_drain_pass` while spinning on a + /// full outbound. A subset of `send_wait`; the remainder is the + /// `tokio::task::yield_now()` await + the (small) cost of + /// `try_send_bytes` itself. + pub coop_drain_in_spin: Duration, + /// Count of `try_send_bytes` calls that returned `Ok(false)` (full). + pub spin_iters: u64, +} + +/// A [`crate::PartitionSink`] over one [`MppSender`]: the produce loop's per-partition send end. +/// `send` runs the cooperative-drain spin and `finish` flushes the channel EOF the same way, so a +/// non-Flight embedder (the in-process harness, pg_search's worker loop) wraps each routed sender +/// in one of these and pushes batches through the trait instead of touching `MppSender` directly. +pub struct MppPartitionSink { + sender: MppSender, + stats: SendBatchStats, +} + +impl MppPartitionSink { + pub fn new(sender: MppSender) -> Self { + Self { + sender, + stats: SendBatchStats::default(), + } + } + + /// Per-channel send counters, for an embedder that traces throughput. Read them before + /// `finish`, which consumes the sink. + pub fn stats(&self) -> &SendBatchStats { + &self.stats + } +} + +#[async_trait] +impl crate::PartitionSink for MppPartitionSink { + async fn send(&mut self, batch: &RecordBatch) -> datafusion::common::Result<()> { + self.sender.send_batch_traced(batch, &mut self.stats).await + } + + async fn finish(mut self: Box) -> datafusion::common::Result<()> { + self.sender.send_eof_traced(&mut self.stats).await + } + + fn cancelled(&self) -> bool { + self.sender.stream_cancelled() + } +} + +/// High-level receiver: pulls bytes via the underlying channel and decodes them +/// into `RecordBatch`. Used by the drain thread. +pub(super) struct MppReceiver { + channel: Box, + /// In-progress oversized frames, keyed by the chunk headers' `(sender_proc, stage_id, + /// partition)`. One entry per stream: a sender's frames for one stream are sequential, + /// so its chunks never interleave with each other, while chunked frames of two streams + /// from the same proc may. A prefix abandoned mid-frame (the producer observed the + /// stream's cancel) sits here until the stream's `Eof` clears it or a fresh frame for + /// the stream restarts at offset 0. + assemblies: Mutex>, +} + +/// One stream's partially reassembled oversized frame. +struct FrameAssembly { + total_len: usize, + buf: Vec, +} + +impl MppReceiver { + pub fn new(channel: Box) -> Self { + Self { + channel, + assemblies: Mutex::new(HashMap::default()), + } + } + + pub(super) fn try_recv_batch(&self) -> RecvBatchOutcome { + loop { + match self.channel.try_recv() { + RecvOutcome::Bytes(bytes) => { + let complete = match self.ingest(bytes) { + Ok(Some(frame)) => frame, + // Absorbed one chunk of a larger frame; pull the next message. + Ok(None) => continue, + Err(e) => return RecvBatchOutcome::Error(e), + }; + return match decode_frame(&complete) { + Ok((header, FrameBody::Batch(batch))) => { + RecvBatchOutcome::Batch { header, batch } + } + Ok((header, FrameBody::Eof)) => RecvBatchOutcome::Eof { header }, + Ok((header, FrameBody::WorkUnit(unit))) => { + RecvBatchOutcome::WorkUnit { header, unit } + } + Ok((header, FrameBody::FeedEof)) => RecvBatchOutcome::FeedEof { header }, + Ok((header, FrameBody::TaskMetrics(metrics))) => { + RecvBatchOutcome::TaskMetrics { header, metrics } + } + Ok((header, FrameBody::SetPlan(frame))) => { + RecvBatchOutcome::SetPlan { header, frame } + } + Ok((header, FrameBody::Cancel)) => RecvBatchOutcome::Cancel { header }, + Err(e) => RecvBatchOutcome::Error(e), + }; + } + RecvOutcome::Empty => return RecvBatchOutcome::Empty, + RecvOutcome::Detached => return RecvBatchOutcome::Detached, + } + } + } + + /// Route one raw ring message through reassembly. Complete frames pass through untouched; + /// `Chunk` frames accumulate per stream until `total_len` bytes arrived, at which point the + /// reassembled inner frame returns for normal decoding. + fn ingest(&self, bytes: Vec) -> Result>, DataFusionError> { + let header = MppFrameHeader::parse(&bytes)?; + let kind = header.kind()?; + if kind != MppFrameKind::Chunk { + if kind == MppFrameKind::Eof { + // The stream is over; drop any prefix its producer abandoned mid-frame. + self.assemblies.lock().unwrap().remove(&( + header.sender_proc(), + header.stage_id, + header.partition, + )); + } + return Ok(Some(bytes)); + } + let (total_len, offset, data) = parse_chunk_frame(&bytes)?; + if total_len < MPP_FRAME_HEADER_SIZE + || offset + .checked_add(data.len()) + .is_none_or(|end| end > total_len) + { + return Err(DataFusionError::Internal(format!( + "mpp: chunk bounds exceed the frame ({offset} + {} > {total_len})", + data.len() + ))); + } + let key = (header.sender_proc(), header.stage_id, header.partition); + let mut assemblies = self.assemblies.lock().unwrap(); + if offset == 0 { + // Start of a new frame; replaces any abandoned predecessor for the same stream. + let mut buf = Vec::new(); + // `try_reserve` so a corrupt total_len surfaces as a query error, not an abort. + buf.try_reserve_exact(total_len).map_err(|e| { + DataFusionError::ResourcesExhausted(format!( + "mpp: cannot buffer a {total_len}-byte reassembled frame: {e}" + )) + })?; + buf.extend_from_slice(data); + if buf.len() == total_len { + return Ok(Some(buf)); + } + assemblies.insert(key, FrameAssembly { total_len, buf }); + return Ok(None); + } + let Some(assembly) = assemblies.get_mut(&key) else { + return Err(DataFusionError::Internal(format!( + "mpp: continuation chunk (offset {offset}) with no first chunk for \ + sender {} stage {} partition {}", + key.0, key.1, key.2 + ))); + }; + if assembly.total_len != total_len || assembly.buf.len() != offset { + return Err(DataFusionError::Internal(format!( + "mpp: chunk out of sequence (have {} of {}, got offset {offset} of {total_len})", + assembly.buf.len(), + assembly.total_len + ))); + } + assembly.buf.extend_from_slice(data); + if assembly.buf.len() == total_len { + let assembly = assemblies.remove(&key).expect("assembly present"); + return Ok(Some(assembly.buf)); + } + Ok(None) + } +} + +/// Decoded result of an [`MppReceiver::try_recv_batch`]. Carries the +/// parsed [`MppFrameHeader`] so the drain thread can route the payload to +/// the right `(stage_id, partition)` channel buffer. +#[derive(Debug)] +pub(super) enum RecvBatchOutcome { + Batch { + header: MppFrameHeader, + batch: RecordBatch, + }, + /// A payload-less `Eof` frame for `header.(stage_id, partition)`. The + /// underlying shm_mq queue is still attached. The sender is just + /// signalling that this logical channel is done, so we can EOF + /// per-channel without dropping the whole queue. + Eof { + header: MppFrameHeader, + }, + /// One work unit for the task named by `header.(stage_id, partition=task)`. + WorkUnit { + header: MppFrameHeader, + unit: pb::WorkUnit, + }, + /// Every feed of the task named by the header is done; its channels close. + FeedEof { + header: MppFrameHeader, + }, + /// The plan for the task named by `header.(stage_id, partition=task)`. + SetPlan { + header: MppFrameHeader, + frame: SetPlanFrame, + }, + /// The collected metrics of the task named by the header. + TaskMetrics { + header: MppFrameHeader, + metrics: pb::TaskMetrics, + }, + /// Consumer abandoned the `header.(stage_id, partition)` stream; its producer stops sending it. + Cancel { + header: MppFrameHeader, + }, + Empty, + Detached, + Error(DataFusionError), +} + +/// Per-`(sender_proc, stage_id, partition)` channel buffer registry owned by a cooperative +/// [`DrainHandle`]. The handle may host several cooperative receivers (DSM MPSC inbox + self-loop +/// in-proc), each demultiplexed by the [`MppFrameHeader`] prefix into the same `map`. +/// `try_drain_pass` looks up the right channel buffer on every frame and pushes the payload into +/// it. Consumers waiting on a given key only see frames matching that key. +/// +/// Each entry is a `DrainBuffer::new(1)`: exactly one sender_proc emits frames for any given +/// channel. Per-channel EOF flows via the `Eof` frame demuxed onto the matching buffer; query- +/// teardown unblock flows via [`DrainHandle::cancel_channel_buffers`] from the handle's `Drop`. +#[derive(Default)] +struct ChannelBufferRegistry { + /// Keyed by `(sender_proc, stage_id, partition)`. The unified inbox carries frames + /// from every peer, so each `(stage, partition)` consumer gets its own per-sender + /// buffer. This preserves the implicit "one stream per sender" semantics that + /// `WorkerConnection::execute` consumers rely on. + map: HashMap<(u32, u32, u32), Arc>, + /// Scopes whose receiver detached (or errored) before draining cleanly. Channels fed by a + /// dead scope fail at registration time too, so a consumer that registers after the detach + /// does not wait on a channel nothing will ever fill. + dead_inbox: bool, + dead_self_loop: bool, +} + +/// Which frames a receiver carries, so a detach can fail exactly the channels it feeds. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum ReceiverScope { + /// The proc's DSM inbox: frames from every peer proc. + Inbox, + /// The in-proc self-loop: frames this proc sends itself. + SelfLoop, +} + +/// Per-sender-proc drain: stashes the receivers and polls them inline from the cooperative spin +/// (no background thread), demuxing each frame into a per-`(stage_id, partition)` channel buffer. +/// +/// Inline polling is the production requirement: pgrx's `check_active_thread` guard panics on any +/// pg FFI call (including `shm_mq_receive`) from a non-backend thread, so the drain work has to +/// run on the backend thread. Tests that need a true thread-backed drain use +/// [`ThreadedDrainHandle`] instead. +/// +/// On drop, the handle cancels every channel buffer so any consumer blocked on `try_pop` unblocks +/// with `Eof` — the drain can therefore never outlive its query, even on a panicked teardown. +pub struct DrainHandle { + /// Per-(stage_id, partition) channel buffer registry. Populated lazily on first frame for a + /// channel, or up-front by callers (e.g. `WorkerConnection::execute`) that need a + /// buffer to wait on before any frame arrives. + channel_buffers: Mutex, + /// Receivers owned by the handle and polled inline from `DrainGatherStream::poll_next` via + /// [`Self::try_drain_pass`]. The `Mutex` is for interior mutability: `try_drain_pass(&self)` + /// marks each slot as `None` after observing `Detached` so subsequent passes skip the dead + /// receiver. `BatchChannelReceiver: Send + Sync` makes `Vec>: Sync` + /// already, so the lock is no longer doubling as the `Sync` provider — replacing it with a + /// non-locking primitive would need either an atomic per-slot detached flag or accepting + /// that detached receivers get polled once per pass (fast-returning `Detached`). The lock + /// is uncontended in production (single backend thread) so the marginal cost is in the + /// type system, not the runtime. + coop_receivers: Mutex>>, + /// This proc's index, used to map a channel's `sender_proc` to the receiver scope that + /// feeds it (`SelfLoop` iff `sender_proc == this_proc`). + this_proc: u32, + /// Worker-side destination of `WorkUnit` frames, keyed `(stage_id, task_number)`. Frames + /// arriving before the embedder registers a task's channels buffer in `Pending`; `FeedEof` + /// drops the senders so the consuming feed streams end, the wire analog of Flight closing + /// its coordinator stream. + feed_registry: Mutex, + /// Leader-side destination of `TaskMetrics` frames: `(stage_id, task_number, metrics)`. + task_metrics_tx: tokio::sync::mpsc::UnboundedSender<(u32, u32, pb::TaskMetrics)>, + task_metrics_rx: + Mutex>>, + /// Worker-side destination of `SetPlan` frames, keyed `(stage_id, task_number)`. Same + /// pending-or-waiting shape as the feed registry: a frame arriving before the task asks + /// buffers in `Pending`; a task asking first parks a oneshot the drain fulfills. + set_plan_registry: Mutex, + /// `(stage_id, partition)` streams this proc's consumers abandoned, learned from inbound + /// `Cancel` frames. A producer blocked on a full outbound checks it in its send spin and ends + /// that stream cleanly, so a consumer that stopped reading early doesn't leave it spinning to + /// the statement timeout. + cancelled_streams: Mutex>, +} + +#[derive(Default)] +struct FeedRegistry { + map: HashMap<(u32, u32), FeedSlot>, + /// Set when the inbox scope died: feeds come from a peer proc, so a dead inbox means no + /// further units or `FeedEof` can arrive. Registered channels get the failure pushed in; + /// later registrations fail immediately. + dead: Option, +} + +enum FeedSlot { + /// Frames that arrived before the embedder registered the task's channels. + Pending { + units: Vec, + done: bool, + }, + Active(RemoteWorkUnitFeedTxs), +} + +/// Push one decoded unit into the channel its `(feed id, partition)` names. A missing channel is +/// not an error: the same tolerance the Flight worker applies to its stream (a feed the plan +/// does not declare is dropped). +fn forward_unit(senders: &RemoteWorkUnitFeedTxs, unit: pb::WorkUnit) { + let Ok(id) = deserialize_uuid(&unit.id) else { + return; + }; + let Some(tx) = senders.get(&(id, unit.partition as usize)) else { + return; + }; + // The feed channels carry the protocol's `WorkUnitMsg`, so the decoded proto frame is mapped + // into it the same way the gRPC worker's `decode_work_unit` does. + let _ = tx.send(Ok(WorkUnitMsg { + id, + partition: unit.partition as usize, + body: unit.body, + created_timestamp_unix_nanos: unit.created_timestamp_unix_nanos as usize, + sent_timestamp_unix_nanos: unit.sent_timestamp_unix_nanos as usize, + received_timestamp_unix_nanos: unit.received_timestamp_unix_nanos as usize, + processed_timestamp_unix_nanos: unit.processed_timestamp_unix_nanos as usize, + })); +} + +fn fail_feed_senders(senders: &RemoteWorkUnitFeedTxs, reason: &str) { + for tx in senders.values() { + let _ = tx.send(Err(DataFusionError::Execution(reason.to_string()))); + } +} + +#[derive(Default)] +struct SetPlanRegistry { + map: HashMap<(u32, u32), SetPlanSlot>, + /// Set when the inbox scope died: plans come from the leader's proc, so a dead inbox means + /// no plan can arrive. Parked takers get the failure; later takers fail immediately. + dead: Option, +} + +enum SetPlanSlot { + /// A frame that arrived before the task asked for it. + Pending(SetPlanFrame), + /// A task that asked before its frame arrived. + Waiting(tokio::sync::oneshot::Sender>), +} + +impl DrainHandle { + /// Construct a cooperative drain handle. Channel buffers are populated lazily by + /// [`Self::try_drain_pass`] when a frame arrives, or up-front by [`Self::register_channel`] + /// when a consumer needs a buffer to wait on before any frame has come in. + pub(super) fn cooperative( + this_proc: u32, + receivers: Vec<(ReceiverScope, MppReceiver)>, + ) -> Self { + let wrapped = receivers.into_iter().map(Some).collect(); + let (task_metrics_tx, task_metrics_rx) = tokio::sync::mpsc::unbounded_channel(); + Self { + channel_buffers: Mutex::new(ChannelBufferRegistry::default()), + coop_receivers: Mutex::new(wrapped), + this_proc, + feed_registry: Mutex::new(FeedRegistry::default()), + task_metrics_tx, + task_metrics_rx: Mutex::new(Some(task_metrics_rx)), + set_plan_registry: Mutex::new(SetPlanRegistry::default()), + cancelled_streams: Mutex::new(HashSet::default()), + } + } + + /// Record a `Cancel` frame: the consumer abandoned the `(stage_id, partition)` stream, so this + /// proc's producer of it stops. Idempotent. + fn note_cancel(&self, stage_id: u32, partition: u32) { + self.cancelled_streams + .lock() + .unwrap() + .insert((stage_id, partition)); + } + + /// Whether a consumer cancelled the `(stage_id, partition)` stream. Read by the send spin to + /// end a producer's stream cleanly when the consumer stopped reading. + pub(super) fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { + self.cancelled_streams + .lock() + .unwrap() + .contains(&(stage_id, partition)) + } + + /// Take the receiving end of the `TaskMetrics` frame stream. The embedder (the leader) + /// drains it into its metrics store; the first caller gets it, later calls get `None`. + pub(super) fn take_task_metrics_receiver( + &self, + ) -> Option> { + self.task_metrics_rx.lock().unwrap().take() + } + + /// Install the senders of one task's feed channels, flushing any units that arrived first. + /// If the task's `FeedEof` (or the inbox death) already came through, the senders drop (or + /// fail) immediately so the consuming streams terminate instead of waiting forever. + pub(super) fn register_work_unit_senders( + &self, + stage_id: u32, + task_number: u32, + senders: RemoteWorkUnitFeedTxs, + ) { + let mut registry = self.feed_registry.lock().unwrap(); + if let Some(reason) = ®istry.dead { + fail_feed_senders(&senders, reason); + return; + } + match registry.map.remove(&(stage_id, task_number)) { + Some(FeedSlot::Pending { units, done }) => { + for unit in units { + forward_unit(&senders, unit); + } + if !done { + registry + .map + .insert((stage_id, task_number), FeedSlot::Active(senders)); + } + } + Some(FeedSlot::Active(_)) | None => { + registry + .map + .insert((stage_id, task_number), FeedSlot::Active(senders)); + } + } + } + + fn route_work_unit(&self, stage_id: u32, task_number: u32, unit: pb::WorkUnit) { + let mut registry = self.feed_registry.lock().unwrap(); + if registry.dead.is_some() { + return; + } + match registry.map.get_mut(&(stage_id, task_number)) { + Some(FeedSlot::Active(senders)) => forward_unit(senders, unit), + Some(FeedSlot::Pending { units, .. }) => units.push(unit), + None => { + registry.map.insert( + (stage_id, task_number), + FeedSlot::Pending { + units: vec![unit], + done: false, + }, + ); + } + } + } + + fn close_feeds(&self, stage_id: u32, task_number: u32) { + let mut registry = self.feed_registry.lock().unwrap(); + match registry.map.get_mut(&(stage_id, task_number)) { + Some(FeedSlot::Active(_)) => { + // Dropping the senders is the close: the consuming streams see end-of-input. + registry.map.remove(&(stage_id, task_number)); + } + Some(FeedSlot::Pending { done, .. }) => *done = true, + None => { + registry.map.insert( + (stage_id, task_number), + FeedSlot::Pending { + units: Vec::new(), + done: true, + }, + ); + } + } + } + + /// Route one decoded `SetPlan` frame to whoever asked for `(stage_id, task_number)`, or + /// buffer it until they do. A duplicate for an already-buffered slot keeps the first frame. + fn route_set_plan(&self, stage_id: u32, task_number: u32, frame: SetPlanFrame) { + let mut registry = self.set_plan_registry.lock().unwrap(); + match registry.map.remove(&(stage_id, task_number)) { + Some(SetPlanSlot::Waiting(tx)) => { + let _ = tx.send(Ok(frame)); + } + Some(pending @ SetPlanSlot::Pending(_)) => { + log::debug!( + "mpp: duplicate SetPlan frame for stage {stage_id} task {task_number}; \ + keeping the first" + ); + registry.map.insert((stage_id, task_number), pending); + } + None => { + registry + .map + .insert((stage_id, task_number), SetPlanSlot::Pending(frame)); + } + } + } + + /// Take the plan delivered for `(stage_id, task_number)`, waiting for its `SetPlan` frame if + /// it has not arrived yet. Something on this proc must keep draining (a pump or a + /// cooperative send spin) or the wait starves; same contract as the feed channels. + pub(super) async fn take_set_plan( + &self, + stage_id: u32, + task_number: u32, + ) -> Result { + let rx = { + let mut registry = self.set_plan_registry.lock().unwrap(); + if let Some(reason) = ®istry.dead { + return Err(DataFusionError::Execution(reason.clone())); + } + match registry.map.remove(&(stage_id, task_number)) { + Some(SetPlanSlot::Pending(frame)) => return Ok(frame), + Some(SetPlanSlot::Waiting(_)) => { + return Err(DataFusionError::Internal(format!( + "mpp: two takers for the SetPlan frame of stage {stage_id} task \ + {task_number}" + ))); + } + None => { + let (tx, rx) = tokio::sync::oneshot::channel(); + registry + .map + .insert((stage_id, task_number), SetPlanSlot::Waiting(tx)); + rx + } + } + }; + rx.await.map_err(|_| { + DataFusionError::Execution( + "mpp: transport torn down before this task's plan arrived".to_string(), + ) + })? + } + + fn scope_for_sender(&self, sender_proc: u32) -> ReceiverScope { + if sender_proc == self.this_proc { + ReceiverScope::SelfLoop + } else { + ReceiverScope::Inbox + } + } + + /// Fail every registered channel fed by `scope` that has not completed yet, and remember the + /// scope as dead so later registrations fail too. Channels whose `Eof` already arrived are + /// untouched: a detach after a clean drain is the normal end of life for a ring. + fn fail_scope(&self, scope: ReceiverScope, reason: &str) { + let to_fail = { + let mut guard = self + .channel_buffers + .lock() + .expect("DrainHandle channel_buffers mutex poisoned"); + match scope { + ReceiverScope::Inbox => guard.dead_inbox = true, + ReceiverScope::SelfLoop => guard.dead_self_loop = true, + } + guard + .map + .iter() + .filter(|((sender_proc, _, _), _)| self.scope_for_sender(*sender_proc) == scope) + .map(|(_, buf)| buf.clone()) + .collect::>() + }; + for buf in to_fail { + buf.fail_pending(reason); + } + if scope == ReceiverScope::Inbox { + let mut registry = self.feed_registry.lock().unwrap(); + registry.dead = Some(reason.to_string()); + for (_, slot) in registry.map.drain() { + if let FeedSlot::Active(senders) = slot { + fail_feed_senders(&senders, reason); + } + } + drop(registry); + let mut plans = self.set_plan_registry.lock().unwrap(); + plans.dead = Some(reason.to_string()); + for (_, slot) in plans.map.drain() { + if let SetPlanSlot::Waiting(tx) = slot { + let _ = tx.send(Err(DataFusionError::Execution(reason.to_string()))); + } + } + } + } + + /// Register (or look up) the channel buffer for `(sender_proc, stage_id, partition)`. + /// The returned `Arc` is the canonical destination for frames matching + /// that key: `try_drain_pass` pushes into the same entry on every `Batch { header, .. }` + /// whose `header.sender_proc()` / `stage_id` / `partition` matches. + pub(super) fn register_channel( + &self, + sender_proc: u32, + stage_id: u32, + partition: u32, + ) -> Arc { + let mut guard = self + .channel_buffers + .lock() + .expect("DrainHandle channel_buffers mutex poisoned"); + let scope_dead = match self.scope_for_sender(sender_proc) { + ReceiverScope::Inbox => guard.dead_inbox, + ReceiverScope::SelfLoop => guard.dead_self_loop, + }; + let buf = guard + .map + .entry((sender_proc, stage_id, partition)) + .or_insert_with(|| { + // num_sources stays 1: each (sender_proc, stage, partition) tuple has + // exactly one upstream (the named sender), even though the underlying + // inbox is shared across all senders. + DrainBuffer::new(1) + }) + .clone(); + drop(guard); + if scope_dead { + buf.fail_pending( + "transport receiver detached before this channel's EOF; the producer went away", + ); + } + buf + } + + /// Cancel every registered channel buffer. Called from `Drop` to unblock any consumer waiting on + /// a channel buffer when the handle goes away mid-query. + /// + /// Collects buffer handles under the registry lock, then notifies after releasing + /// it. Notifying inline would block any concurrent [`Self::register_channel`] for + /// as long as it takes to acquire `DrainBuffer::inner` N times. Fine today (single + /// backend thread), but cheap insurance against the multi-thread variant landing + /// later. + fn cancel_channel_buffers(&self) { + let to_cancel = { + let guard = self + .channel_buffers + .lock() + .expect("DrainHandle channel_buffers mutex poisoned"); + guard.map.values().cloned().collect::>() + }; + for buf in to_cancel { + buf.cancel(); + } + } + + /// Pull batches from each live receiver and demux them into the per-`(stage_id, partition)` + /// channel buffer registry. Called from `DrainGatherStream::poll_next` and from + /// `MppSender::send_batch`'s cooperative spin. Drain work happens on the backend thread + /// (pgrx-safe). No-op for thread-backed handles. + /// + /// Each pass drains *every available* batch from each receiver (up to a safety cap). Pulling + /// only one batch per source per call would mean that under steady producer pressure the + /// cooperative sender's spin-loop can't keep up: we'd fall N:1 behind peers' sends and the + /// mesh would stall once any queue fills. Draining until the receiver reports `Empty` bounds + /// each pass by queue depth rather than by spin-loop iteration count. + /// + /// Returns `Ok(())` once every cooperative receiver has been pulled until `Empty` (or + /// detached). Errors propagate as `Err` so a transport-level failure surfaces at the call + /// site rather than getting silently dropped. + /// + /// Routing rules per outcome: + /// - `Batch { header, batch }`: look up (or lazily create) the + /// `(header.stage_id, header.partition)` channel buffer and push `batch`. + /// - `Eof { header }`: per-channel EOF. Resolve the channel buffer and call + /// `notify_source_done`. Other channels on the same queue keep flowing, + /// so the receiver slot stays live. + /// - `Detached` / `Error`: queue-wide shutdown. Notify every registered + /// channel buffer, mark the handle detached, and drop the slot. + pub fn try_drain_pass(&self) -> Result<(), DataFusionError> { + // Bound per-source pulls per call. The upper limit exists to give + // the caller a chance to re-try its own send between drains — + // otherwise a proc with a very fast peer could drain + // indefinitely on one source and starve its own outbound. + const MAX_BATCHES_PER_SOURCE_PER_PASS: usize = 256; + + let mut slots = self.coop_receivers.lock().unwrap(); + for slot in slots.iter_mut() { + let Some((scope, rx)) = slot.as_ref() else { + continue; + }; + let scope = *scope; + for _ in 0..MAX_BATCHES_PER_SOURCE_PER_PASS { + match rx.try_recv_batch() { + RecvBatchOutcome::Batch { header, batch } => { + let buf = self.register_channel( + header.sender_proc(), + header.stage_id, + header.partition, + ); + buf.push_batch(batch); + } + RecvBatchOutcome::Eof { header } => { + let buf = self.register_channel( + header.sender_proc(), + header.stage_id, + header.partition, + ); + buf.notify_source_done(); + // Other channels may still flow on this queue, so the receiver slot + // stays live. + } + RecvBatchOutcome::WorkUnit { header, mut unit } => { + set_received_time(&mut unit); + self.route_work_unit(header.stage_id, header.partition, unit); + } + RecvBatchOutcome::FeedEof { header } => { + self.close_feeds(header.stage_id, header.partition); + } + RecvBatchOutcome::TaskMetrics { header, metrics } => { + // The embedder may have dropped the receiver; metrics are best-effort. + let _ = + self.task_metrics_tx + .send((header.stage_id, header.partition, metrics)); + } + RecvBatchOutcome::SetPlan { header, frame } => { + self.route_set_plan(header.stage_id, header.partition, frame); + } + RecvBatchOutcome::Cancel { header } => { + self.note_cancel(header.stage_id, header.partition); + } + RecvBatchOutcome::Empty => break, + RecvBatchOutcome::Detached => { + // Only THIS receiver is dead. The drain holds multiple receivers + // (own-inbox MPSC + self-loop in-proc); one going away doesn't + // imply the others have. Fail only the channels this receiver's + // scope feeds, and only those still waiting on their `Eof`: after a + // clean drain the detach is the ring's normal end of life, but a + // channel that never got its `Eof` (producer crash, early sender + // drop) would otherwise spin on `try_pop -> None` forever. + *slot = None; + self.fail_scope( + scope, + "transport receiver detached before this channel's EOF; the \ + producer went away", + ); + break; + } + RecvBatchOutcome::Error(e) => { + // Same scoping as Detached, but the ring reported corruption (or the + // receiver poisoned itself), so even completed siblings can't be + // trusted to have been the last word. Still scope-limited: the other + // receiver is an independent transport. The error also propagates to + // this caller directly. + *slot = None; + self.fail_scope(scope, &format!("transport receiver failed: {e}")); + return Err(e); + } + } + } + } + Ok(()) + } +} + +impl Drop for DrainHandle { + fn drop(&mut self) { + // Unblock any consumer blocked on a channel buffer when the handle is torn down before EOF + // flows naturally (e.g. a query error en route to ExecEndCustomScan). + self.cancel_channel_buffers(); + } +} +/// SPSC channel pair for two use cases: +/// - Unit tests (bounded capacity, exercising backpressure). +/// - Production self-loop slots: when a worker's fragment emits a partition destined for +/// its OWN proc (e.g. peer-mesh hash routing where consumer task t lands on the same +/// worker as producer task t), the DSM layout has no self-pair inbox: a process is +/// not its own peer. The dispatcher routes those self-loops through this in-proc +/// channel, which exposes the same `BatchChannelSender` / `BatchChannelReceiver` +/// surface as the DSM ring so the drain and channel-buffer registry don't need a +/// special case. +/// +/// Production callers pass a very large `capacity` so the channel is effectively unbounded under +/// steady state. The current-thread Tokio runtime interleaves producer and consumer fragments +/// via `yield_now().await`, so backpressure would be benign anyway, but unbounded rules out any +/// chance of self-deadlock if the producer never yields. +pub(super) fn in_proc_channel(capacity: usize) -> (InProcSender, InProcReceiver) { + let (tx, rx) = std::sync::mpsc::sync_channel::>(capacity); + ( + InProcSender { + tx, + send_lock: tokio::sync::Mutex::new(()), + }, + InProcReceiver { rx: Mutex::new(rx) }, + ) +} + +pub(super) struct InProcSender { + tx: std::sync::mpsc::SyncSender>, + /// Per-instance lock so the [`BatchChannelSender::send_lock`] contract holds even when an + /// in-proc channel ends up in a code path that would otherwise need serialization. In-proc + /// `send_bytes` is already atomic (each call pushes a complete `Vec`), so the lock is + /// effectively a no-op here; keeping it uniform with `DsmInboxSender` avoids + /// special-casing the caller. + send_lock: tokio::sync::Mutex<()>, +} + +pub(super) struct InProcReceiver { + // The std::sync::mpsc receiver is !Sync; wrap in a Mutex so the drain + // thread can hold it behind a `Box` (which is + // `Send + Sync`-relaxed by design, but we only need Send for the thread + // hand-off). Tests only ever access from one thread so the Mutex is + // uncontended. + rx: Mutex>>, +} + +impl BatchChannelSender for InProcSender { + fn send_bytes(&self, bytes: &[u8]) -> Result<(), DataFusionError> { + self.tx.send(bytes.to_vec()).map_err(|_| { + DataFusionError::Execution("mpp: in-proc channel detached during send".into()) + }) + } + + fn try_send_bytes(&self, bytes: &[u8]) -> Result { + match self.tx.try_send(bytes.to_vec()) { + Ok(()) => Ok(true), + Err(std::sync::mpsc::TrySendError::Full(_)) => Ok(false), + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => Err(DataFusionError::Execution( + "mpp: in-proc channel detached during try_send".into(), + )), + } + } + + fn send_lock(&self) -> &tokio::sync::Mutex<()> { + &self.send_lock + } +} + +impl BatchChannelReceiver for InProcReceiver { + fn try_recv(&self) -> RecvOutcome { + let rx = self.rx.lock().expect("InProcReceiver mutex poisoned"); + match rx.try_recv() { + Ok(bytes) => RecvOutcome::Bytes(bytes), + Err(std::sync::mpsc::TryRecvError::Empty) => RecvOutcome::Empty, + Err(std::sync::mpsc::TryRecvError::Disconnected) => RecvOutcome::Detached, + } + } +} + +/// Effectively unbounded capacity for self-loop in-proc channels. The +/// `std::sync::mpsc::sync_channel` API requires a numeric capacity; this constant picks one large +/// enough that production workloads won't reach it but small enough that a runaway producer +/// (e.g. infinite-loop bug) won't allocate billions of `Vec` before OOM. +pub(super) const SELF_LOOP_CAPACITY: usize = 1 << 20; + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::{ + Int32Array, Int64Array, StringArray, StringViewArray, UInt64Array, + }; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use std::sync::Arc as StdArc; + use std::thread; + + use std::thread::JoinHandle; + + impl DrainBuffer { + /// Block until a batch is available, EOF is reached, or the buffer is cancelled. + /// + /// INVARIANT: any already-buffered batch is returned *before* honoring either + /// cancellation or all-sources-done. Reordering the queue pop ahead of the cancel/eof + /// check would silently drop buffered data on an otherwise-clean shutdown; the + /// `drain_buffer_drains_buffered_before_eof` test locks this in. + fn pop_front(&self) -> DrainItem { + let mut guard = self.inner.lock().expect("DrainBuffer mutex poisoned"); + loop { + if let Some(batch) = guard.queue.pop_front() { + return DrainItem::Batch(batch); + } + if let Some(msg) = &guard.failed { + return DrainItem::Failed(msg.clone()); + } + if guard.cancelled || guard.sources_done >= guard.num_sources { + return DrainItem::Eof; + } + guard = self.cond.wait(guard).expect("DrainBuffer mutex poisoned"); + } + } + + /// True if `cancel` has been called. The local `drain_loop` consults this; the + /// cooperative production path watches the flag through `notify_source_done` fan-out + /// instead. + fn is_cancelled(&self) -> bool { + self.inner + .lock() + .expect("DrainBuffer mutex poisoned") + .cancelled + } + } + + impl MppSender { + /// Construct a sender with the default `(stage_id=0, partition=0)` header. Used where + /// the header carries no actionable routing info. + fn new(channel: Arc) -> Self { + Self::with_header(channel, MppFrameHeader::batch(0, 0, 0)) + } + + /// Stats-less wrapper around `send_batch_traced`. Production call sites + /// (`ShuffleStream::process_batch`) always pass a `SendBatchStats` so per-peer + /// wall-time shows up in the EOF trace. Wraps the async send in a tiny current-thread + /// Tokio runtime so `#[test]` functions don't have to be `#[tokio::test]` and the + /// OS-thread-spawning test harnesses don't have to plumb an async runtime themselves. + fn send_batch(&self, batch: &RecordBatch) -> Result<(), DataFusionError> { + let mut stats = SendBatchStats::default(); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("test tokio runtime build"); + rt.block_on(self.send_batch_traced(batch, &mut stats)) + } + } + + /// Configuration for `spawn_drain_thread`. pgrx panics on any pg FFI call (including + /// `shm_mq_receive`) from a non-backend thread, so production never spawns a drain thread — + /// see [`DrainHandle::cooperative`] for the cooperative path. + struct DrainConfig { + /// Receivers to drain. Ownership moves into the spawned thread. + receivers: Vec, + /// Destination buffer. + buffer: Arc, + /// How long to sleep when every receiver is empty but some are still attached. Tuning: + /// small values reduce end-of-batch latency but raise CPU; 1 ms is a safe default until + /// we integrate with WaitLatch. + idle_sleep: Duration, + } + + impl DrainConfig { + fn new(receivers: Vec, buffer: Arc) -> Self { + Self { + receivers, + buffer, + idle_sleep: Duration::from_millis(1), + } + } + } + + /// Spawn the dedicated drain thread. The thread round-robins through every receiver with + /// non-blocking `try_recv`, pushes decoded batches into `buffer`, and marks each source done + /// as soon as it observes a detach or decode error. When every source is done, the thread + /// exits. + fn spawn_drain_thread(config: DrainConfig) -> JoinHandle> { + thread::spawn(move || drain_loop(config)) + } + + /// RAII wrapper: owns the drain thread's `JoinHandle` and the buffer it writes into. + /// `Drop` cancels the buffer (unblocking the consumer) and joins the thread, so the thread + /// can never outlive the test scope even on a panic. + struct ThreadedDrainHandle { + buffer: Arc, + join: Mutex>>>, + } + + impl ThreadedDrainHandle { + fn spawn(config: DrainConfig) -> Self { + let buffer = Arc::clone(&config.buffer); + let join = spawn_drain_thread(config); + Self { + buffer, + join: Mutex::new(Some(join)), + } + } + } + + impl Drop for ThreadedDrainHandle { + fn drop(&mut self) { + self.buffer.cancel(); + if let Some(join) = self.join.lock().unwrap().take() { + let _ = join.join(); + } + } + } + + /// Test-only thread-backed drain. Writes every observed frame into a single shared + /// [`DrainBuffer`] with `num_sources = N`. Per-channel `Eof` frames are treated as "this source + /// is done" rather than "this logical channel within the source is done"; sufficient for unit + /// tests that don't exercise per-channel demux. Production drains route through + /// [`DrainHandle::try_drain_pass`] (cooperative variant), which keys on the frame header. Tests + /// that need to validate production demux must use [`DrainHandle::cooperative`] and call + /// `try_drain_pass` directly. + fn drain_loop(config: DrainConfig) -> Result<(), DataFusionError> { + let DrainConfig { + receivers, + buffer, + idle_sleep, + } = config; + + let mut done = vec![false; receivers.len()]; + loop { + // Observe cancellation before each pass so a `DrainHandle::drop` with + // live peer senders tears down cleanly. Without this check, the drain + // thread would spin `try_recv` forever because no source has detached. + if buffer.is_cancelled() { + return Ok(()); + } + + let mut got_any = false; + let mut all_done = true; + for (i, rx) in receivers.iter().enumerate() { + if done[i] { + continue; + } + all_done = false; + match rx.try_recv_batch() { + RecvBatchOutcome::Batch { header: _, batch } => { + got_any = true; + buffer.push_batch(batch); + } + RecvBatchOutcome::Eof { header: _ } => { + // Per-channel Eof frame: single-channel positional design + // treats it as a source-done signal. See `try_drain_pass`. + done[i] = true; + buffer.notify_source_done(); + } + // Control frames have no place in the test-only single-buffer path. + RecvBatchOutcome::WorkUnit { .. } + | RecvBatchOutcome::FeedEof { .. } + | RecvBatchOutcome::TaskMetrics { .. } + | RecvBatchOutcome::SetPlan { .. } + | RecvBatchOutcome::Cancel { .. } => {} + RecvBatchOutcome::Empty => {} + RecvBatchOutcome::Detached => { + done[i] = true; + buffer.notify_source_done(); + } + RecvBatchOutcome::Error(e) => { + // Treat a decode error as a fatal detach for this source + // so the consumer can observe EOF and abort the query. + done[i] = true; + buffer.notify_source_done(); + return Err(e); + } + } + } + + if all_done { + return Ok(()); + } + if !got_any { + thread::sleep(idle_sleep); + } + } + } + + fn sample_batch(rows: i32) -> RecordBatch { + let schema = StdArc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + ])); + let ids = Int32Array::from_iter_values(0..rows); + let names = StringArray::from_iter_values((0..rows).map(|i| format!("n{i}"))); + RecordBatch::try_new(schema, vec![StdArc::new(ids), StdArc::new(names)]).unwrap() + } + + #[test] + fn frame_round_trips_a_batch_with_header() { + let orig = sample_batch(64); + let header = MppFrameHeader::batch(7, 3, 0); + let mut buf = Vec::with_capacity(1024); + encode_frame_into(header, &orig, &mut buf).expect("encode_frame"); + + let (parsed, body) = decode_frame(&buf).expect("decode_frame"); + assert_eq!(parsed, header); + assert_eq!(parsed.kind().unwrap(), MppFrameKind::Batch); + let FrameBody::Batch(decoded) = body else { + panic!("Batch frame must carry a batch payload"); + }; + assert_eq!(decoded.num_rows(), 64); + assert_eq!(decoded.schema(), orig.schema()); + assert_eq!(decoded.num_columns(), orig.num_columns()); + for col in 0..orig.num_columns() { + assert_eq!(orig.column(col).as_ref(), decoded.column(col).as_ref()); + } + } + + #[test] + fn frame_round_trips_eof() { + let mut buf = Vec::new(); + encode_eof_frame_into(2, 5, 0, &mut buf).expect("encode_eof"); + assert_eq!(buf.len(), MPP_FRAME_HEADER_SIZE); + + let (header, body) = decode_frame(&buf).expect("decode_frame"); + assert_eq!(header, MppFrameHeader::eof(2, 5, 0)); + assert_eq!(header.kind().unwrap(), MppFrameKind::Eof); + assert!(matches!(body, FrameBody::Eof)); + } + + #[test] + fn frame_round_trips_cancel() { + let header = MppFrameHeader::cancel(4, 2, 0); + let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; + header.write_to(&mut buf); + + let (parsed, body) = decode_frame(&buf).expect("decode_frame"); + assert_eq!(parsed, header); + assert_eq!(parsed.kind().unwrap(), MppFrameKind::Cancel); + assert_eq!(parsed.stage_id, 4); + assert_eq!(parsed.partition, 2); + assert!(matches!(body, FrameBody::Cancel)); + } + + #[test] + fn drain_records_cancel_scoped_to_its_stream() { + let drain = DrainHandle::cooperative(0, Vec::new()); + assert!(!drain.stream_cancelled(7, 1)); + drain.note_cancel(7, 1); + assert!(drain.stream_cancelled(7, 1)); + // A cancel for one stream leaves the others alive, the way gRPC closes one stream: same + // stage but a different partition, and a different stage, both stay live. + assert!(!drain.stream_cancelled(7, 2)); + assert!(!drain.stream_cancelled(8, 1)); + } + + /// The producer-side half of the early-termination fix: a producer blocked on a full outbound + /// ends its send cleanly once a `Cancel` for its `(stage, partition)` stream lands on its inbox. + /// The test hangs if the spin doesn't observe the cancel. + #[tokio::test(flavor = "current_thread")] + async fn producer_send_ends_when_consumer_cancels_the_stream() { + // Outbound to a consumer that never drains: capacity 1, so the second send finds it full. + let (out_tx, _out_rx) = in_proc_channel(1); + // The producer's inbox, where the consumer's `Cancel` frame lands. + let (inbox_tx, inbox_rx) = in_proc_channel(4); + let drain = Arc::new(DrainHandle::cooperative( + 1, + vec![(ReceiverScope::Inbox, MppReceiver::new(Box::new(inbox_rx)))], + )); + // Producer of the `(stage 7, partition 0)` stream. + let sender = MppSender::with_header(Arc::new(out_tx), MppFrameHeader::batch(7, 0, 1)) + .with_cooperative_drain(Arc::clone(&drain) as Arc); + + // The consumer abandons that stream (it stopped reading before EOF): a `Cancel` reaches the + // inbox. + let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; + MppFrameHeader::cancel(7, 0, 0).write_to(&mut buf); + inbox_tx.send_bytes(&buf).unwrap(); + + let mut stats = SendBatchStats::default(); + // First batch fills the one slot. + sender + .send_batch_traced(&sample_batch(1), &mut stats) + .await + .unwrap(); + // The second batch would block forever on the full slot. Instead the spin drains the inbox, + // sees the cancel, and ends the send cleanly. + sender + .send_batch_traced(&sample_batch(1), &mut stats) + .await + .unwrap(); + } + + #[test] + fn frame_round_trips_a_work_unit_and_feed_eof() { + let unit = pb::WorkUnit { + id: vec![7; 16], + partition: 4, + body: vec![1, 2, 3], + created_timestamp_unix_nanos: 11, + sent_timestamp_unix_nanos: 22, + received_timestamp_unix_nanos: 0, + processed_timestamp_unix_nanos: 0, + }; + let header = MppFrameHeader::work_unit(3, 1, 0); + let mut buf = Vec::new(); + encode_prost_frame_into(header, &unit, &mut buf).expect("encode work unit"); + let (parsed, body) = decode_frame(&buf).expect("decode work unit"); + assert_eq!(parsed, header); + let FrameBody::WorkUnit(decoded) = body else { + panic!("WorkUnit frame must carry a unit"); + }; + assert_eq!(decoded, unit); + + let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; + MppFrameHeader::feed_eof(3, 1, 0).write_to(&mut buf); + let (parsed, body) = decode_frame(&buf).expect("decode feed eof"); + assert_eq!(parsed.kind().unwrap(), MppFrameKind::FeedEof); + assert!(matches!(body, FrameBody::FeedEof)); + } + + #[test] + fn frame_round_trips_task_metrics() { + let metrics = pb::TaskMetrics { + pre_order_plan_metrics: vec![], + task_metrics: None, + }; + let header = MppFrameHeader::task_metrics(2, 0, 1); + let mut buf = Vec::new(); + encode_prost_frame_into(header, &metrics, &mut buf).expect("encode metrics"); + let (parsed, body) = decode_frame(&buf).expect("decode metrics"); + assert_eq!(parsed, header); + assert!(matches!(body, FrameBody::TaskMetrics(m) if m == metrics)); + } + + #[test] + fn work_units_buffer_until_registration_and_feed_eof_closes() { + fn unit(partition: u64) -> pb::WorkUnit { + pb::WorkUnit { + id: vec![9; 16], + partition, + body: vec![], + created_timestamp_unix_nanos: 0, + sent_timestamp_unix_nanos: 0, + received_timestamp_unix_nanos: 0, + processed_timestamp_unix_nanos: 0, + } + } + let drain = DrainHandle::cooperative(1, vec![]); + + // Units arriving before registration must buffer, not drop. + drain.route_work_unit(5, 0, unit(0)); + drain.route_work_unit(5, 0, unit(0)); + + let id = crate::common::deserialize_uuid(&[9; 16]).unwrap(); + let mut channels = crate::work_unit_feed::RemoteWorkUnitFeedRegistry::default(); + channels.add(id, 1); + let mut rx = channels + .receivers + .get(&(id, 0)) + .unwrap() + .lock() + .unwrap() + .take() + .unwrap(); + drain.register_work_unit_senders(5, 0, channels.senders); + + assert!(rx.try_recv().unwrap().is_ok()); + assert!(rx.try_recv().unwrap().is_ok()); + // Still open: the producer may send more units. + assert!(rx.try_recv().is_err()); + + // FeedEof drops the senders, which ends the stream. + drain.route_work_unit(5, 0, unit(0)); + drain.close_feeds(5, 0); + assert!(rx.try_recv().unwrap().is_ok()); + assert!(matches!( + rx.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + } + + #[test] + fn frame_round_trips_a_set_plan_with_headers() { + let set_plan = pb::SetPlanRequest { + plan_proto: vec![1, 2, 3, 4], + task_count: 2, + task_key: Some(pb::TaskKey { + query_id: vec![5; 16], + stage_id: 3, + task_number: 1, + }), + work_unit_feed_declarations: vec![], + target_worker_url: "inprocess://worker/1".to_string(), + query_start_time_ns: 42, + }; + let mut headers = http::HeaderMap::new(); + headers.insert("x-datafusion-distributed-config", "abc".parse().unwrap()); + headers.append("x-repeated", "one".parse().unwrap()); + headers.append("x-repeated", "two".parse().unwrap()); + + let frame = SetPlanFrame::from_parts(set_plan.clone(), &headers).expect("from_parts"); + let header = MppFrameHeader::set_plan(3, 1, 0); + let mut buf = Vec::new(); + encode_prost_frame_into(header, &frame, &mut buf).expect("encode set plan"); + let (parsed, body) = decode_frame(&buf).expect("decode set plan"); + assert_eq!(parsed, header); + assert_eq!(parsed.kind().unwrap(), MppFrameKind::SetPlan); + let FrameBody::SetPlan(decoded) = body else { + panic!("SetPlan frame must carry a SetPlanFrame"); + }; + let (decoded_plan, decoded_headers) = decoded.into_parts().expect("into_parts"); + assert_eq!(decoded_plan, set_plan); + assert_eq!(decoded_headers, headers); + } + + fn sample_set_plan_frame(plan_proto: Vec) -> SetPlanFrame { + SetPlanFrame { + set_plan: Some(pb::SetPlanRequest { + plan_proto, + task_count: 1, + task_key: None, + work_unit_feed_declarations: vec![], + target_worker_url: String::new(), + query_start_time_ns: 0, + }), + header_keys: vec![], + header_values: vec![], + } + } + + #[tokio::test] + async fn set_plan_serves_taker_in_either_arrival_order() { + let drain = DrainHandle::cooperative(1, vec![]); + + // Frame first: the take resolves from the pending slot. + drain.route_set_plan(7, 0, sample_set_plan_frame(vec![1])); + let frame = drain.take_set_plan(7, 0).await.expect("pending take"); + assert_eq!(frame.set_plan.unwrap().plan_proto, vec![1]); + + // Taker first: the frame fulfills the parked oneshot. + let take = drain.take_set_plan(7, 1); + futures::pin_mut!(take); + assert!(futures::poll!(take.as_mut()).is_pending()); + drain.route_set_plan(7, 1, sample_set_plan_frame(vec![2])); + let frame = take.await.expect("waiting take"); + assert_eq!(frame.set_plan.unwrap().plan_proto, vec![2]); + } + + #[tokio::test] + async fn set_plan_take_fails_when_the_inbox_dies() { + let drain = DrainHandle::cooperative(1, vec![]); + let take = drain.take_set_plan(9, 0); + futures::pin_mut!(take); + assert!(futures::poll!(take.as_mut()).is_pending()); + drain.fail_scope(ReceiverScope::Inbox, "producer went away"); + let err = take.await.expect_err("dead inbox must fail the take"); + assert!(format!("{err}").contains("producer went away")); + // Later takers fail immediately. + let err = drain.take_set_plan(9, 1).await.expect_err("dead registry"); + assert!(format!("{err}").contains("producer went away")); + } + + #[test] + fn frame_rejects_short_message() { + let too_short = vec![0u8; MPP_FRAME_HEADER_SIZE - 1]; + let err = decode_frame(&too_short).expect_err("short frame must fail"); + assert!(format!("{err}").contains("too short")); + } + + #[test] + fn frame_rejects_bad_magic() { + // Explicit non-zero, non-magic prefix. Don't rely on the + // happenstance that 0u32 != MPP_FRAME_MAGIC. + let mut bad = vec![0u8; MPP_FRAME_HEADER_SIZE]; + bad[0..4].copy_from_slice(&0xCAFEBABE_u32.to_le_bytes()); + let err = decode_frame(&bad).expect_err("bad magic must fail"); + assert!(format!("{err}").contains("bad frame magic")); + bad[0..4].copy_from_slice(&0xDEADBEEF_u32.to_le_bytes()); + let err = decode_frame(&bad).expect_err("bad magic must fail"); + assert!(format!("{err}").contains("bad frame magic")); + } + + #[test] + fn frame_rejects_unknown_kind() { + let header = MppFrameHeader { + magic: MPP_FRAME_MAGIC, + flags: 0x42, // unknown kind byte, no reserved bits set + stage_id: 0, + partition: 0, + }; + let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; + header.write_to(&mut buf); + let err = decode_frame(&buf).expect_err("unknown kind must fail"); + assert!(format!("{err}").contains("unknown frame kind")); + } + + #[test] + fn frame_rejects_reserved_flag_bits() { + // Reserved range is bits 8..16. Bits 16..32 are sender_proc and must NOT trip the + // reserved check. Cover both boundaries of the reserved range. + for bit in [0x0000_0100u32, 0x0000_8000u32] { + let header = MppFrameHeader { + magic: MPP_FRAME_MAGIC, + flags: bit, // kind byte 0 (Batch), reserved bit set, no sender_proc + stage_id: 0, + partition: 0, + }; + let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; + header.write_to(&mut buf); + let err = decode_frame(&buf).expect_err(&format!("reserved bit {bit:#x} must fail")); + assert!( + format!("{err}").contains("reserved frame flag bits"), + "bit {bit:#x}: {err}" + ); + } + } + + #[test] + fn frame_kind_coexists_with_max_sender_proc() { + // Negative-space companion to frame_rejects_reserved_flag_bits: setting every bit in + // 16..32 (= max sender_proc) along with kind=Eof in bit 0 must parse cleanly without + // tripping the reserved-bits check, and sender_proc()/kind() must read both back. + let header = MppFrameHeader { + magic: MPP_FRAME_MAGIC, + flags: 0xFFFF_0001, // Eof in low byte, max sender_proc in high half, reserved=0 + stage_id: 0, + partition: 0, + }; + assert_eq!(header.kind().unwrap(), MppFrameKind::Eof); + assert_eq!(header.sender_proc(), MPP_MAX_SENDER_PROC); + } + + #[test] + fn frame_sender_proc_round_trip() { + // sender_proc lives in flags bits 16..32 and shouldn't collide with kind or reserved. + for &sp in &[0u32, 1, 7, 255, 256, 1023, 65534, MPP_MAX_SENDER_PROC] { + let header = MppFrameHeader::batch(11, 5, sp); + assert_eq!(header.sender_proc(), sp, "batch round-trip sp={sp}"); + assert_eq!(header.kind().unwrap(), MppFrameKind::Batch); + + let mut buf = Vec::with_capacity(MPP_FRAME_HEADER_SIZE); + let payload = sample_batch(8); + encode_frame_into(header, &payload, &mut buf).expect("encode batch"); + let (parsed, _) = decode_frame(&buf).expect("decode batch"); + assert_eq!(parsed.sender_proc(), sp, "decoded batch sender_proc"); + + let mut eof_buf = Vec::new(); + encode_eof_frame_into(11, 5, sp, &mut eof_buf).expect("encode eof"); + let (parsed_eof, _) = decode_frame(&eof_buf).expect("decode eof"); + assert_eq!(parsed_eof.sender_proc(), sp, "decoded eof sender_proc"); + assert_eq!(parsed_eof.kind().unwrap(), MppFrameKind::Eof); + } + } + + #[test] + fn frame_eof_with_payload_is_rejected() { + let mut buf = Vec::with_capacity(32); + encode_eof_frame_into(0, 0, 0, &mut buf).expect("encode_eof"); + buf.push(0xAB); // smuggle a payload byte after the Eof header + let err = decode_frame(&buf).expect_err("Eof+payload must fail"); + assert!(format!("{err}").contains("payload-less frame carries payload")); + } + + #[test] + fn codec_round_trips_many_batch_sizes() { + let mut buf = Vec::with_capacity(1024); + for rows in [0, 1, 7, 64, 1024] { + let orig = sample_batch(rows); + encode_frame_into(MppFrameHeader::batch(0, 0, 0), &orig, &mut buf).expect("encode"); + let (_header, body) = decode_frame(&buf).expect("decode"); + let FrameBody::Batch(decoded) = body else { + panic!("Batch frame must carry a batch payload"); + }; + assert_eq!(orig.num_rows(), decoded.num_rows()); + } + } + + /// Bounded in-proc sink standing in for a DSM ring: rejects frames over `cap` the way + /// `try_send` does, records every accepted frame, and advertises the bound so the send + /// path's chunker engages. + struct BoundedSink { + cap: usize, + frames: Mutex>>, + send_lock: tokio::sync::Mutex<()>, + } + + impl BoundedSink { + fn new(cap: usize) -> Arc { + Arc::new(Self { + cap, + frames: Mutex::new(Vec::new()), + send_lock: tokio::sync::Mutex::new(()), + }) + } + } + + impl BatchChannelSender for BoundedSink { + fn send_bytes(&self, bytes: &[u8]) -> Result<(), DataFusionError> { + if bytes.len() > self.cap { + return Err(DataFusionError::Execution( + "mpp: DSM MPSC frame exceeds the entire ring capacity \ + (raise the embedder's queue-size knob)" + .into(), + )); + } + self.frames.lock().unwrap().push(bytes.to_vec()); + Ok(()) + } + + fn send_lock(&self) -> &tokio::sync::Mutex<()> { + &self.send_lock + } + + fn max_frame_bytes(&self) -> Option { + Some(self.cap) + } + } + + /// `rows` of ~1 KiB unique strings, so slices drag a large values buffer through + /// arrow-ipc. + fn wide_batch(rows: usize) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("s", DataType::Utf8, false), + Field::new("v", DataType::Int64, false), + ])); + let strings = StringArray::from_iter_values( + (0..rows).map(|i| format!("{i:06}-{}", "x".repeat(1024))), + ); + let vals = Int64Array::from_iter_values(0..rows as i64); + RecordBatch::try_new(schema, vec![Arc::new(strings), Arc::new(vals)]).unwrap() + } + + /// Replays recorded frames as a `BatchChannelReceiver`, standing in for the ring's recv + /// side so reassembly can be exercised without a DSM region. + struct ReplayChannel { + frames: Mutex>>, + } + + impl ReplayChannel { + fn over(frames: Vec>) -> Box { + Box::new(Self { + frames: Mutex::new(frames.into_iter().collect()), + }) + } + } + + impl BatchChannelReceiver for ReplayChannel { + fn try_recv(&self) -> RecvOutcome { + match self.frames.lock().unwrap().pop_front() { + Some(f) => RecvOutcome::Bytes(f), + None => RecvOutcome::Empty, + } + } + } + + /// Pull one decoded batch out of a receiver fed by `frames`, asserting nothing else is + /// queued behind it. + fn reassemble_single_batch(frames: Vec>) -> (MppFrameHeader, RecordBatch) { + let receiver = MppReceiver::new(ReplayChannel::over(frames)); + let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { + panic!("expected a reassembled batch"); + }; + assert!(matches!(receiver.try_recv_batch(), RecvBatchOutcome::Empty)); + (header, batch) + } + + #[test] + fn oversized_sliced_batch_compacts_then_chunks() { + let cap = 64 * 1024; + let base = wide_batch(512); + // 128 rows, but the slice keeps the whole ~512 KiB values buffer alive. + let sliced = base.slice(64, 128); + + // Premise: the sliced encode really does balloon past the cap. If arrow-ipc ever + // learns to truncate sliced buffers, this assert flags the test (and the compact + // step's main motivation) for review. + let mut probe = Vec::new(); + encode_frame_into(MppFrameHeader::batch(3, 7, 0), &sliced, &mut probe).expect("probe"); + assert!( + probe.len() > cap, + "premise: sliced encode ({} bytes) must exceed cap ({cap})", + probe.len() + ); + + let sink = BoundedSink::new(cap); + let sender = MppSender::with_header( + Arc::clone(&sink) as Arc, + MppFrameHeader::batch(3, 7, 0), + ); + sender.send_batch(&sliced).expect("chunked send"); + + // The compacted 128 rows (~135 KiB of strings) still exceed the cap, so the frame + // streams as chunks, every ring message within the cap. + let frames = sink.frames.lock().unwrap().clone(); + assert!( + frames.len() > 1, + "expected the oversized frame to chunk, got {} frame(s)", + frames.len() + ); + for frame in &frames { + assert!( + frame.len() <= cap, + "frame of {} bytes over cap", + frame.len() + ); + let header = MppFrameHeader::parse(frame).expect("chunk header"); + assert_eq!(header.kind().expect("kind"), MppFrameKind::Chunk); + assert_eq!((header.stage_id, header.partition), (3, 7)); + } + let (header, got) = reassemble_single_batch(frames); + assert_eq!((header.stage_id, header.partition), (3, 7)); + // Row order and content must survive; compare against a compacted copy (`take` of + // every row) since the decoded side never sees the original buffers. + let want = compact_rows(&sliced, 0, sliced.num_rows()).expect("compact reference"); + assert_eq!(got, want); + } + + /// Same as above but through `Utf8View`: `take` copies the views yet keeps the shared + /// data blocks, so without the compact-time `gc` every frame would drag the full data + /// blocks through the ring no matter how it is chunked. + #[test] + fn oversized_view_array_batch_compacts_then_chunks() { + let cap = 64 * 1024; + let schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Utf8View, + false, + )])); + let strings = StringViewArray::from_iter_values( + (0..512).map(|i| format!("{i:06}-{}", "x".repeat(1024))), + ); + let base = RecordBatch::try_new(schema, vec![Arc::new(strings)]).unwrap(); + let sliced = base.slice(64, 128); + + let sink = BoundedSink::new(cap); + let sender = MppSender::with_header( + Arc::clone(&sink) as Arc, + MppFrameHeader::batch(1, 1, 0), + ); + sender.send_batch(&sliced).expect("view-array chunked send"); + + let frames = sink.frames.lock().unwrap().clone(); + for frame in &frames { + assert!( + frame.len() <= cap, + "frame of {} bytes over cap", + frame.len() + ); + } + let (_, got) = reassemble_single_batch(frames); + assert_eq!(got.num_rows(), 128); + } + + /// A single row bigger than the whole ring streams through in chunks; frame size is + /// never an error. + #[test] + fn single_oversized_row_streams_through() { + let cap = 64 * 1024; + let value = "y".repeat(128 * 1024); + let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from_iter_values([value.as_str()]))], + ) + .unwrap(); + + let sink = BoundedSink::new(cap); + let sender = MppSender::with_header( + Arc::clone(&sink) as Arc, + MppFrameHeader::batch(0, 0, 0), + ); + sender.send_batch(&batch).expect("giant row streams"); + + let frames = sink.frames.lock().unwrap().clone(); + assert!( + frames.len() > 2, + "a 128 KiB row needs several 64 KiB chunks" + ); + let (_, got) = reassemble_single_batch(frames); + assert_eq!(got.num_rows(), 1); + let strings = got + .column(0) + .as_any() + .downcast_ref::() + .expect("string column"); + assert_eq!(strings.value(0), value); + } + + /// Chunked frames from two streams of one proc may interleave on the shared inbox; the + /// per-stream reassembly keys keep them apart, and a complete frame passes through the + /// middle of an assembly untouched. + #[test] + fn chunk_reassembly_keys_by_stream() { + let sender_proc = 2; + let inner_a = { + let mut buf = Vec::new(); + encode_frame_into( + MppFrameHeader::batch(1, 1, sender_proc), + &sample_batch(4), + &mut buf, + ) + .expect("encode inner A"); + buf + }; + let chunk_header = MppFrameHeader::chunk(1, 1, sender_proc); + let split_at = inner_a.len() / 2; + let mut a0 = Vec::new(); + encode_chunk_frame_into( + chunk_header, + inner_a.len() as u64, + 0, + &inner_a[..split_at], + &mut a0, + ); + let mut a1 = Vec::new(); + encode_chunk_frame_into( + chunk_header, + inner_a.len() as u64, + split_at as u64, + &inner_a[split_at..], + &mut a1, + ); + let mut b = Vec::new(); + encode_frame_into( + MppFrameHeader::batch(1, 2, sender_proc), + &sample_batch(7), + &mut b, + ) + .expect("encode B"); + + let receiver = MppReceiver::new(ReplayChannel::over(vec![a0, b, a1])); + // B lands first: its complete frame doesn't wait on A's assembly. + let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { + panic!("expected B"); + }; + assert_eq!((header.partition, batch.num_rows()), (2, 7)); + let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { + panic!("expected reassembled A"); + }; + assert_eq!((header.partition, batch.num_rows()), (1, 4)); + assert!(matches!(receiver.try_recv_batch(), RecvBatchOutcome::Empty)); + } + + /// A producer that observes its stream's cancel stops mid-frame; the stream's `Eof` + /// clears the orphaned prefix, and a later frame for the stream restarts at offset 0. + #[test] + fn eof_clears_abandoned_chunk_prefix() { + let sender_proc = 3; + let mut inner = Vec::new(); + encode_frame_into( + MppFrameHeader::batch(5, 0, sender_proc), + &sample_batch(4), + &mut inner, + ) + .expect("encode inner"); + let chunk_header = MppFrameHeader::chunk(5, 0, sender_proc); + let mut partial = Vec::new(); + encode_chunk_frame_into( + chunk_header, + inner.len() as u64, + 0, + &inner[..inner.len() / 2], + &mut partial, + ); + let mut eof = Vec::new(); + encode_eof_frame_into(5, 0, sender_proc, &mut eof).expect("encode eof"); + // A fresh frame for the stream after the abandonment restarts cleanly at offset 0. + let mut whole = Vec::new(); + encode_chunk_frame_into(chunk_header, inner.len() as u64, 0, &inner, &mut whole); + + let receiver = MppReceiver::new(ReplayChannel::over(vec![partial, eof, whole])); + let RecvBatchOutcome::Eof { header } = receiver.try_recv_batch() else { + panic!("expected the Eof to pass through"); + }; + assert_eq!((header.stage_id, header.partition), (5, 0)); + assert!( + receiver.assemblies.lock().unwrap().is_empty(), + "the Eof must clear the abandoned prefix" + ); + assert!(matches!( + receiver.try_recv_batch(), + RecvBatchOutcome::Batch { .. } + )); + } + + /// Reassembly rejects a continuation with no first chunk and an offset that doesn't + /// line up: both mean the per-sender FIFO the protocol rides on was violated. + #[test] + fn chunk_out_of_sequence_errors() { + let sender_proc = 4; + let chunk_header = MppFrameHeader::chunk(1, 0, sender_proc); + let mut orphan = Vec::new(); + encode_chunk_frame_into(chunk_header, 1000, 500, &[0u8; 16], &mut orphan); + let receiver = MppReceiver::new(ReplayChannel::over(vec![orphan])); + assert!(matches!( + receiver.try_recv_batch(), + RecvBatchOutcome::Error(_) + )); + + let mut first = Vec::new(); + encode_chunk_frame_into(chunk_header, 1000, 0, &[0u8; 100], &mut first); + let mut skipped = Vec::new(); + encode_chunk_frame_into(chunk_header, 1000, 200, &[0u8; 100], &mut skipped); + let receiver = MppReceiver::new(ReplayChannel::over(vec![first, skipped])); + assert!(matches!( + receiver.try_recv_batch(), + RecvBatchOutcome::Error(_) + )); + } + + #[test] + fn drain_buffer_pop_returns_pushed_batches_in_order() { + let buf = DrainBuffer::new(1); + buf.push_batch(sample_batch(3)); + buf.push_batch(sample_batch(5)); + buf.notify_source_done(); + + match buf.pop_front() { + DrainItem::Batch(b) => assert_eq!(b.num_rows(), 3), + DrainItem::Eof => panic!("expected batch"), + DrainItem::Failed(msg) => panic!("unexpected failure: {msg}"), + } + match buf.pop_front() { + DrainItem::Batch(b) => assert_eq!(b.num_rows(), 5), + DrainItem::Eof => panic!("expected batch"), + DrainItem::Failed(msg) => panic!("unexpected failure: {msg}"), + } + matches!(buf.pop_front(), DrainItem::Eof); + } + + #[test] + fn drain_buffer_pop_blocks_until_push_then_eof() { + let buf = DrainBuffer::new(2); + let producer = StdArc::clone(&buf); + let handle = thread::spawn(move || { + thread::sleep(Duration::from_millis(20)); + producer.push_batch(sample_batch(2)); + producer.notify_source_done(); + thread::sleep(Duration::from_millis(20)); + producer.notify_source_done(); + }); + + match buf.pop_front() { + DrainItem::Batch(b) => assert_eq!(b.num_rows(), 2), + DrainItem::Eof => panic!("expected batch first"), + DrainItem::Failed(msg) => panic!("unexpected failure: {msg}"), + } + assert!(matches!(buf.pop_front(), DrainItem::Eof)); + handle.join().unwrap(); + } + + #[test] + fn drain_buffer_cancel_unblocks_waiter() { + let buf = DrainBuffer::new(1); + let canceller = StdArc::clone(&buf); + let handle = thread::spawn(move || { + thread::sleep(Duration::from_millis(20)); + canceller.cancel(); + }); + assert!(matches!(buf.pop_front(), DrainItem::Eof)); + handle.join().unwrap(); + } + + #[test] + fn in_proc_channel_round_trips_through_mpp_sender_receiver() { + let (tx, rx) = in_proc_channel(8); + let sender = MppSender::new(Arc::new(tx)); + let receiver = MppReceiver::new(Box::new(rx)); + + sender.send_batch(&sample_batch(4)).unwrap(); + std::mem::drop(sender); + + match receiver.try_recv_batch() { + RecvBatchOutcome::Batch { header: _, batch } => assert_eq!(batch.num_rows(), 4), + other => panic!("expected batch, got {other:?}"), + } + assert!(matches!( + receiver.try_recv_batch(), + RecvBatchOutcome::Detached + )); + } + + #[test] + fn drain_thread_drains_single_source() { + let (tx, rx) = in_proc_channel(4); + let sender = MppSender::new(Arc::new(tx)); + let receiver = MppReceiver::new(Box::new(rx)); + let buffer = DrainBuffer::new(1); + + let join = spawn_drain_thread(DrainConfig::new(vec![receiver], StdArc::clone(&buffer))); + + thread::spawn(move || { + for rows in [1, 2, 3, 4, 5] { + sender.send_batch(&sample_batch(rows)).unwrap(); + } + // Drop sender to signal EOF + }) + .join() + .unwrap(); + + let mut received = Vec::new(); + while let DrainItem::Batch(b) = buffer.pop_front() { + received.push(b.num_rows()); + } + assert_eq!(received, vec![1, 2, 3, 4, 5]); + join.join().unwrap().unwrap(); + } + + #[test] + fn drain_handle_shutdown_joins_cleanly() { + let (tx, rx) = in_proc_channel(4); + let sender = MppSender::new(Arc::new(tx)); + let receiver = MppReceiver::new(Box::new(rx)); + let buffer = DrainBuffer::new(1); + let handle = + ThreadedDrainHandle::spawn(DrainConfig::new(vec![receiver], StdArc::clone(&buffer))); + + sender.send_batch(&sample_batch(2)).unwrap(); + std::mem::drop(sender); // detach + // Pop the one batch + assert!(matches!(buffer.pop_front(), DrainItem::Batch(_))); + assert!(matches!(buffer.pop_front(), DrainItem::Eof)); + // Drop drives production teardown (cancel + join). Test passes if + // this returns without hanging. + std::mem::drop(handle); + } + + #[test] + fn drain_handle_drop_cancels_and_joins() { + // Build a drain that never detaches (we keep the sender alive), then + // drop the handle. The Drop impl must cancel the buffer and join the + // thread without hanging. + let (tx, rx) = in_proc_channel(4); + let _sender_kept_alive = MppSender::new(Arc::new(tx)); + let receiver = MppReceiver::new(Box::new(rx)); + let buffer = DrainBuffer::new(1); + let handle = + ThreadedDrainHandle::spawn(DrainConfig::new(vec![receiver], StdArc::clone(&buffer))); + + // Simulate consumer path error: drop the handle without calling + // shutdown(). The drain thread must exit before drop returns. + let start = Instant::now(); + drop(handle); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_secs(2), + "ThreadedDrainHandle::drop took too long: {elapsed:?}" + ); + // Consumer observes EOF because cancel was called. + assert!(matches!(buffer.pop_front(), DrainItem::Eof)); + } + + #[test] + fn drain_thread_drains_n2_mesh_100k_batches() { + // Simulates a 2-proc mesh under load. Each of two producers + // pushes 50_000 small batches through a bounded channel; the drain + // thread interleaves and the consumer reads EOF exactly after + // receiving all 100_000 batches. Exercises backpressure (bounded + // capacity = 16) without deadlock. + const PER_SOURCE: usize = 50_000; + let (tx0, rx0) = in_proc_channel(16); + let (tx1, rx1) = in_proc_channel(16); + let receivers = vec![ + MppReceiver::new(Box::new(rx0)), + MppReceiver::new(Box::new(rx1)), + ]; + let buffer = DrainBuffer::new(2); + let drain_join = spawn_drain_thread(DrainConfig::new(receivers, StdArc::clone(&buffer))); + + let tx0_send = MppSender::new(Arc::new(tx0)); + let tx1_send = MppSender::new(Arc::new(tx1)); + let batch_template = sample_batch(1); + + let p0 = { + let b = batch_template.clone(); + thread::spawn(move || { + for _ in 0..PER_SOURCE { + tx0_send.send_batch(&b).unwrap(); + } + }) + }; + let p1 = { + let b = batch_template.clone(); + thread::spawn(move || { + for _ in 0..PER_SOURCE { + tx1_send.send_batch(&b).unwrap(); + } + }) + }; + + let mut total = 0usize; + while let DrainItem::Batch(_) = buffer.pop_front() { + total += 1; + } + assert_eq!(total, 2 * PER_SOURCE); + p0.join().unwrap(); + p1.join().unwrap(); + drain_join.join().unwrap().unwrap(); + } + + #[test] + fn drain_buffer_drains_buffered_before_eof() { + // Even if all sources have finished and cancel fires, any already- + // buffered batches must be observed before Eof. + let buf = DrainBuffer::new(1); + buf.push_batch(sample_batch(1)); + buf.push_batch(sample_batch(1)); + buf.notify_source_done(); + buf.cancel(); + + assert!(matches!(buf.pop_front(), DrainItem::Batch(_))); + assert!(matches!(buf.pop_front(), DrainItem::Batch(_))); + assert!(matches!(buf.pop_front(), DrainItem::Eof)); + } + + // --------------------------------------------------------------------- + // Throughput microbenches. + // + // These are `#[ignore]` by default because they spin for seconds and spam stdout. Run with: + // + // cargo test --package pg_search --release \ + // postgres::customscan::mpp::transport::tests::throughput \ + // -- --ignored --nocapture + // + // They help us bound the transport layer's cost independently of DataFusion/Tantivy. All use + // the `in_proc_channel` backend (same `MppSender`/`MppReceiver` trait boundary as the shm_mq + // one), so numbers here are an optimistic ceiling. shm_mq adds the ring-buffer copy + + // cross-process notification cost on top. If these numbers are already below the row rate + // the real query needs, we know IPC encode + channel handoff is the bottleneck without + // needing CI data. + // --------------------------------------------------------------------- + + /// Row shape matching the post-Partial shuffle in + /// `aggregate_join_groupby`: a grouping key (title string) plus two + /// partial-aggregate accumulators (COUNT u64, SUM i64). + fn postagg_shape_batch(rows: usize) -> RecordBatch { + let schema = StdArc::new(Schema::new(vec![ + Field::new("title", DataType::Utf8, false), + Field::new("count_partial", DataType::UInt64, false), + Field::new("sum_partial", DataType::Int64, false), + ])); + // Titles averaging ~30 bytes, typical for the docs dataset. + let titles = StringArray::from_iter_values( + (0..rows).map(|i| format!("file_{i:012}_title_with_some_length")), + ); + let counts = UInt64Array::from_iter_values((0..rows as u64).map(|i| i % 64 + 1)); + let sums = Int64Array::from_iter_values((0..rows as i64).map(|i| i * 1024)); + RecordBatch::try_new( + schema, + vec![StdArc::new(titles), StdArc::new(counts), StdArc::new(sums)], + ) + .unwrap() + } + + /// Row shape matching the probe-side shuffle in the same query: + /// `pages.fileId` (u64) plus `pages.sizeInBytes` (i64). + fn probe_shape_batch(rows: usize) -> RecordBatch { + let schema = StdArc::new(Schema::new(vec![ + Field::new("fileId", DataType::UInt64, false), + Field::new("sizeInBytes", DataType::Int64, false), + ])); + let ids = + UInt64Array::from_iter_values((0..rows as u64).map(|i| i.wrapping_mul(2654435761))); + let sizes = Int64Array::from_iter_values((0..rows as i64).map(|i| i * 37)); + RecordBatch::try_new(schema, vec![StdArc::new(ids), StdArc::new(sizes)]).unwrap() + } + + fn bench_throughput( + label: &str, + make_batch: fn(usize) -> RecordBatch, + batch_rows: usize, + total_rows: usize, + ) { + let batches = total_rows.div_ceil(batch_rows); + let template = make_batch(batch_rows); + // Encode once up front so we also report pure-encode throughput + // separately. Real queries encode inside the hot path per batch. + let enc_start = Instant::now(); + let mut enc_bytes = 0usize; + let mut enc_buf = Vec::with_capacity(1024); + for _ in 0..batches { + encode_frame_into(MppFrameHeader::batch(0, 0, 0), &template, &mut enc_buf) + .expect("encode"); + enc_bytes += enc_buf.len(); + } + let enc_elapsed = enc_start.elapsed(); + + // N=2 mesh: two senders, one drain thread, one consumer. Matches + // the gb_postagg / gb_right topology in the real query. + let (tx0, rx0) = in_proc_channel(16); + let (tx1, rx1) = in_proc_channel(16); + let receivers = vec![ + MppReceiver::new(Box::new(rx0)), + MppReceiver::new(Box::new(rx1)), + ]; + let buffer = DrainBuffer::new(2); + let drain_join = spawn_drain_thread(DrainConfig::new(receivers, StdArc::clone(&buffer))); + let tx0_send = MppSender::new(Arc::new(tx0)); + let tx1_send = MppSender::new(Arc::new(tx1)); + + let per_source = batches / 2; + let round_trip_start = Instant::now(); + let p0 = { + let b = template.clone(); + thread::spawn(move || { + for _ in 0..per_source { + tx0_send.send_batch(&b).unwrap(); + } + }) + }; + let p1 = { + let b = template.clone(); + thread::spawn(move || { + for _ in 0..per_source { + tx1_send.send_batch(&b).unwrap(); + } + }) + }; + + let mut got_rows = 0usize; + let mut got_batches = 0usize; + while let DrainItem::Batch(b) = buffer.pop_front() { + got_rows += b.num_rows(); + got_batches += 1; + } + p0.join().unwrap(); + p1.join().unwrap(); + drain_join.join().unwrap().unwrap(); + let rt_elapsed = round_trip_start.elapsed(); + + let enc_mb_per_s = (enc_bytes as f64 / (1024.0 * 1024.0)) / enc_elapsed.as_secs_f64(); + let enc_rows_per_s = (batches * batch_rows) as f64 / enc_elapsed.as_secs_f64(); + let rt_rows_per_s = got_rows as f64 / rt_elapsed.as_secs_f64(); + let rt_bytes_total_mb = enc_bytes as f64 / (1024.0 * 1024.0); + let rt_mb_per_s = rt_bytes_total_mb / rt_elapsed.as_secs_f64(); + let per_batch_us = rt_elapsed.as_micros() as f64 / got_batches as f64; + + println!( + "[throughput] {label:<18} batch_rows={batch_rows:<5} batches={got_batches:<6} rows={got_rows} \ + encode_only: {enc_rows_per_s:>11.0} rows/s {enc_mb_per_s:>7.1} MB/s | \ + round_trip: {rt_rows_per_s:>11.0} rows/s {rt_mb_per_s:>7.1} MB/s ({per_batch_us:.1}us/batch)" + ); + } + + #[test] + #[ignore] + fn throughput_postagg_shape() { + // Sweeps batch size to show per-batch fixed cost vs per-row cost. + // 1.25M total rows ≈ what one proc ships through gb_postagg at + // 25M scale. 625K per proc × 2 = 1.25M. + for batch_rows in [128, 512, 2048, 8192, 32_768] { + bench_throughput("postagg", postagg_shape_batch, batch_rows, 1_250_000); + } + } + + #[test] + #[ignore] + fn throughput_probe_shape() { + // 12.5M total rows ≈ what one proc ships through gb_right at 25M. + for batch_rows in [128, 512, 2048, 8192, 32_768] { + bench_throughput("probe", probe_shape_batch, batch_rows, 12_500_000); + } + } + + // --------------------------------------------------------------------- + // Per-`(stage_id, partition)` channel buffer registry on the cooperative `DrainHandle`. + // + // Producers stamp `MppFrameHeader::batch(stage_id, partition)` on every outgoing frame, and + // the receiver-side cooperative drain demuxes by header into a channel buffer per + // `(stage_id, partition)`. These tests use the `in_proc_channel` backend to drive + // `try_drain_pass` from the test thread. That mirrors how the production path runs the drain + // inline from `DrainGatherStream::poll_next` on the backend thread. + // --------------------------------------------------------------------- + + /// Drain a `DrainHandle::cooperative` to completion: poll until every receiver returns + /// `Empty`. With the `in_proc_channel` test backend the drain observes `Detached` once the + /// producer drops its sender, so a bounded loop of `try_drain_pass` calls is enough to flush + /// everything the producer wrote. + fn drain_until_detached(handle: &DrainHandle) { + for _ in 0..64 { + handle.try_drain_pass().expect("try_drain_pass"); + } + } + + #[test] + fn drain_handle_demuxes_frames_by_header() { + // One queue carrying two channels: `(0, 0)` and `(0, 1)`. Each + // channel buffer receives only its own batches. Per-channel EOF is out of scope + // here. See `drain_handle_eof_frame_closes_one_channel` for explicit-Eof routing + // and `drain_handle_drop_cancels_registered_channel_buffers` for the + // teardown-EOF contract. + let (tx, rx) = in_proc_channel(8); + let base = MppSender::new(Arc::new(tx)); + let s00 = base.clone_with_header(MppFrameHeader::batch(0, 0, 0)); + let s01 = base.clone_with_header(MppFrameHeader::batch(0, 1, 0)); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + s00.send_batch(&sample_batch(2)).unwrap(); + s01.send_batch(&sample_batch(7)).unwrap(); + s00.send_batch(&sample_batch(3)).unwrap(); + drop(s00); + drop(s01); + drop(base); + + let buf00 = handle.register_channel(0, 0, 0); + let buf01 = handle.register_channel(0, 0, 1); + + drain_until_detached(&handle); + + let mut p0_rows = Vec::new(); + while let Some(DrainItem::Batch(b)) = buf00.try_pop() { + p0_rows.push(b.num_rows()); + } + let mut p1_rows = Vec::new(); + while let Some(DrainItem::Batch(b)) = buf01.try_pop() { + p1_rows.push(b.num_rows()); + } + assert_eq!(p0_rows, vec![2, 3]); + assert_eq!(p1_rows, vec![7]); + } + + #[test] + fn drain_handle_eof_frame_closes_one_channel() { + // An `Eof` frame on `(0, 0)` closes that channel buffer while frames on + // `(0, 1)` continue to flow on the same queue. `Detached` doesn't broadcast a + // registry-wide EOF, so `(0, 1)` surfaces EOF only when the handle's `Drop` + // runs `cancel_channel_buffers`. + let (tx, rx) = in_proc_channel(8); + let tx_arc: Arc = Arc::new(tx); + let s00 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 0, 0)); + let s01 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 1, 0)); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + s00.send_batch(&sample_batch(4)).unwrap(); + let mut eof_buf = Vec::new(); + encode_eof_frame_into(0, 0, 0, &mut eof_buf).unwrap(); + tx_arc.send_bytes(&eof_buf).unwrap(); + s01.send_batch(&sample_batch(6)).unwrap(); + + let buf00 = handle.register_channel(0, 0, 0); + let buf01 = handle.register_channel(0, 0, 1); + + drop(s00); + drop(s01); + drop(tx_arc); + drain_until_detached(&handle); + + match buf00.try_pop() { + Some(DrainItem::Batch(b)) => assert_eq!(b.num_rows(), 4), + other => panic!("expected (0,0) batch, got {other:?}"), + } + assert!(matches!(buf00.try_pop(), Some(DrainItem::Eof))); + + match buf01.try_pop() { + Some(DrainItem::Batch(b)) => assert_eq!(b.num_rows(), 6), + other => panic!("expected (0,1) batch, got {other:?}"), + } + assert!(buf01.try_pop().is_none()); + drop(handle); + assert!(matches!(buf01.try_pop(), Some(DrainItem::Eof))); + } + + #[test] + fn drain_handle_register_channel_is_idempotent() { + // Two calls for the same key return Arcs pointing to the same + // DrainBuffer instance. + let (_tx, rx) = in_proc_channel(8); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + let first = handle.register_channel(0, 2, 3); + let second = handle.register_channel(0, 2, 3); + assert!(Arc::ptr_eq(&first, &second)); + } + + #[test] + fn drain_handle_demuxes_frames_by_stage_id() { + // Same partition (0) for two different stage ids on the same queue. + // The registry's compound key keeps them on separate channel buffers. + let (tx, rx) = in_proc_channel(8); + let tx_arc: Arc = Arc::new(tx); + let s_stage0 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 0, 0)); + let s_stage1 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(1, 0, 0)); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + s_stage0.send_batch(&sample_batch(2)).unwrap(); + s_stage1.send_batch(&sample_batch(9)).unwrap(); + s_stage0.send_batch(&sample_batch(4)).unwrap(); + drop(s_stage0); + drop(s_stage1); + drop(tx_arc); + + let buf0 = handle.register_channel(0, 0, 0); + let buf1 = handle.register_channel(0, 1, 0); + + drain_until_detached(&handle); + + let mut stage0_rows = Vec::new(); + while let Some(DrainItem::Batch(b)) = buf0.try_pop() { + stage0_rows.push(b.num_rows()); + } + let mut stage1_rows = Vec::new(); + while let Some(DrainItem::Batch(b)) = buf1.try_pop() { + stage1_rows.push(b.num_rows()); + } + assert_eq!(stage0_rows, vec![2, 4]); + assert_eq!(stage1_rows, vec![9]); + } + + #[test] + fn drain_handle_drop_cancels_registered_channel_buffers() { + // Dropping a cooperative DrainHandle must wake any consumer holding an Arc + // from `register_channel`. Otherwise a query error path that tears down the mesh would + // leave a consumer blocked on a buffer that will never see EOF. + let (_tx, rx) = in_proc_channel(8); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + let buf_a = handle.register_channel(0, 0, 0); + let buf_b = handle.register_channel(0, 7, 3); + // No data ever flows; the handle is just dropped. + drop(handle); + + assert!(matches!(buf_a.try_pop(), Some(DrainItem::Eof))); + assert!(matches!(buf_b.try_pop(), Some(DrainItem::Eof))); + } +} diff --git a/src/work_unit_feed/mod.rs b/src/work_unit_feed/mod.rs index ec74f52ce..f51eb6430 100644 --- a/src/work_unit_feed/mod.rs +++ b/src/work_unit_feed/mod.rs @@ -5,6 +5,7 @@ mod work_unit_feed; mod work_unit_feed_provider; mod work_unit_feed_registry; +pub(crate) use remote_work_unit_feed::RemoteWorkUnitFeedTxs; pub(crate) use remote_work_unit_feed::{ RemoteWorkUnitFeedRegistry, build_work_unit_batch_msg, set_work_unit_received_time, set_work_unit_send_time, From d851dff266de0cc33a976864cb7e7c7f2baf21ad Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Tue, 14 Jul 2026 15:41:54 -0700 Subject: [PATCH 08/15] Incorporate the `paradedb/actions` CI jobs, and remove the docs-deploy workflow. Squashed from the following commits: * Incorporate the `paradedb/actions` rebase sync job. (#32, #33) * Run CI on `target-patch-*` branches. (#35) * Removed the docs-deploy GitHub Pages workflow. * Add snapshot tagging job (#38) * Fix Slack mentions by always mentioning at-most the approvers (#41) * ci: bump paradedb/actions to v8 for the upstream sync (#42) * ci: Add explicit permissions to the sync workflows (#43) * Use actions v10 for upstream sync alerts (#54) --- .github/workflows/ci.yml | 1 + .github/workflows/docs.yml | 55 ---------------------- .github/workflows/snapshot.yml | 44 +++++++++++++++++ .github/workflows/sync-promote-branch.yml | 35 ++++++++++++++ .github/workflows/sync-upstream-rebase.yml | 34 +++++++++++++ scripts/sync-upstream.sh | 29 ++++++++++++ 6 files changed, 143 insertions(+), 55 deletions(-) delete mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/snapshot.yml create mode 100644 .github/workflows/sync-promote-branch.yml create mode 100644 .github/workflows/sync-upstream-rebase.yml create mode 100755 scripts/sync-upstream.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5748469b6..5cda50ae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - 'target-patch-*' env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 78f827a0e..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Deploy Documentation - -on: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: true - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: '3.x' - - - name: Install dependencies - run: | - pip install -r docs/requirements.txt - - - name: Build documentation - run: | - cd docs - make html - - - name: Upload artifact - uses: actions/upload-pages-artifact@v4 - with: - path: docs/build/html - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 000000000..e8847ad8b --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,44 @@ +name: Cut Snapshot Tag + +on: + workflow_dispatch: + +jobs: + snapshot: + name: Create Snapshot Tag + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Generate GitHub App Token + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.PARADEDB_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.PARADEDB_GITHUB_APP_PRIVATE_KEY }} + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + persist-credentials: false + + - name: Configure Git + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + git config --global user.name "paradedb-github-app[bot]" + git config --global user.email "282009505+paradedb-github-app[bot]@users.noreply.github.com" + git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + + - name: Create and Push Tag + run: | + set -euo pipefail + TAG_NAME="snapshot-${{ github.ref_name }}-$(date -u +%Y-%m-%d-%H%M%S)" + echo "Creating tag: $TAG_NAME" + + git tag "$TAG_NAME" + git push origin "$TAG_NAME" + + echo "✅ Successfully created and pushed tag $TAG_NAME" diff --git a/.github/workflows/sync-promote-branch.yml b/.github/workflows/sync-promote-branch.yml new file mode 100644 index 000000000..1fb2d6b72 --- /dev/null +++ b/.github/workflows/sync-promote-branch.yml @@ -0,0 +1,35 @@ +# workflows/sync-promote-branch.yml +# +# Promote Target Patch Branch to Main +# Promote a resolved target-patch-* branch to main after manual approval and CI validation. + +name: Promote Target Patch Branch to Main + +on: + workflow_dispatch: + inputs: + branch_name: + description: "Branch name to promote (must match target-patch-*)" + required: true + type: string + +concurrency: + group: promote-branch + cancel-in-progress: false + +permissions: + actions: write + contents: write + issues: write + pull-requests: read + +jobs: + call-promote: + uses: paradedb/actions/.github/workflows/upstream-sync-promote.yml@v10 + with: + branch_name: ${{ inputs.branch_name }} + approvers: "philippemnoel,rebasedming,stuhood,mdashti" + github_app_client_id: ${{ vars.PARADEDB_GITHUB_APP_CLIENT_ID }} + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }} + PARADEDB_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PARADEDB_GITHUB_APP_PRIVATE_KEY }} diff --git a/.github/workflows/sync-upstream-rebase.yml b/.github/workflows/sync-upstream-rebase.yml new file mode 100644 index 000000000..a3720a7df --- /dev/null +++ b/.github/workflows/sync-upstream-rebase.yml @@ -0,0 +1,34 @@ +# workflows/sync-upstream-rebase.yml +# +# Upstream Rebase +# Periodically rebase the target patches onto the latest upstream commits and promote the result to main. + +name: Upstream Rebase + +on: + schedule: + - cron: "0 14,16,18,20,22 * * 1-5" # Mon-Fri 6am-2pm PST (UTC weekdays) + - cron: "0 0,2 * * 2-6" # Mon-Fri 4pm-6pm PST (next UTC day) + workflow_dispatch: # Allow manual triggering + +concurrency: + group: upstream-rebase + cancel-in-progress: false + +permissions: + actions: write + checks: read + contents: write + issues: write + pull-requests: read + +jobs: + call-rebase: + uses: paradedb/actions/.github/workflows/upstream-sync-rebase.yml@v10 + with: + github_app_client_id: ${{ vars.PARADEDB_GITHUB_APP_CLIENT_ID }} + approvers: "philippemnoel,rebasedming,stuhood,mdashti" + slack_alert_mention: "" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }} + PARADEDB_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PARADEDB_GITHUB_APP_PRIVATE_KEY }} diff --git a/scripts/sync-upstream.sh b/scripts/sync-upstream.sh new file mode 100755 index 000000000..87cb5dcb7 --- /dev/null +++ b/scripts/sync-upstream.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# scripts/sync-upstream.sh +# +# Wrapper script to execute the centralized upstream sync logic. + +set -euo pipefail + +# 1. Define repository-specific configuration +export UPSTREAM_REPO="datafusion-contrib/datafusion-distributed" +export UPSTREAM_REPO_URL="https://github.com/datafusion-contrib/datafusion-distributed.git" +export TARGET_REPO="paradedb/datafusion-distributed" +export TARGET_BRANCH="main" +export UPSTREAM_BRANCH="main" + +# 2. Define the URL to the centralized script +# Using the raw content URL from the central repository +CORE_SCRIPT_URL="https://raw.githubusercontent.com/paradedb/actions/v10/upstream-sync/scripts/sync-core.sh" + +# 3. Download and source the core logic as an API +TMP_SCRIPT=$(mktemp) +curl -fsSL "$CORE_SCRIPT_URL" -o "$TMP_SCRIPT" +# shellcheck source=/dev/null +source "$TMP_SCRIPT" +rm -f "$TMP_SCRIPT" + +# 4. Only execute the command router if run directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + sync_core_main "$@" +fi From 30a9153ee6f42ec5a7eff45905208e503b827f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20No=C3=ABl?= <21990816+philippemnoel@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:45:56 -0700 Subject: [PATCH 09/15] Use the datafusion-maintainers team as the promotion roster (#64) I had forgotten this one when moving to GitHub Teams --- .github/workflows/sync-promote-branch.yml | 2 +- .github/workflows/sync-upstream-rebase.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-promote-branch.yml b/.github/workflows/sync-promote-branch.yml index 1fb2d6b72..1473d3db4 100644 --- a/.github/workflows/sync-promote-branch.yml +++ b/.github/workflows/sync-promote-branch.yml @@ -28,7 +28,7 @@ jobs: uses: paradedb/actions/.github/workflows/upstream-sync-promote.yml@v10 with: branch_name: ${{ inputs.branch_name }} - approvers: "philippemnoel,rebasedming,stuhood,mdashti" + approvers: datafusion-maintainers github_app_client_id: ${{ vars.PARADEDB_GITHUB_APP_CLIENT_ID }} secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }} diff --git a/.github/workflows/sync-upstream-rebase.yml b/.github/workflows/sync-upstream-rebase.yml index a3720a7df..14eae27e0 100644 --- a/.github/workflows/sync-upstream-rebase.yml +++ b/.github/workflows/sync-upstream-rebase.yml @@ -27,7 +27,7 @@ jobs: uses: paradedb/actions/.github/workflows/upstream-sync-rebase.yml@v10 with: github_app_client_id: ${{ vars.PARADEDB_GITHUB_APP_CLIENT_ID }} - approvers: "philippemnoel,rebasedming,stuhood,mdashti" + approvers: datafusion-maintainers slack_alert_mention: "" secrets: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GITHUB_CHANNEL_WEBHOOK_URL }} From dde6c2d3aed15b37a4528f758d679583d9057023 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Thu, 6 Aug 2026 12:25:54 +0900 Subject: [PATCH 10/15] chore: Move the shm module to a pull-based (RPC) model (#62) As described in the new `README.md`, this change moves the `shm` module from push based to pull based. In practice, this means that when a worker starts up, it idles until it receives the new `ExecuteTaskFrame` message over the wire (corresponding to the same message on the gRPC side), and only then begins executing a particular task. ---- This does _not_ actually result in a reduction in lines of code. But I'm confident that better aligning with gRPC will allow us to more easily fix https://github.com/paradedb/paradedb/pull/5809 ... and https://github.com/paradedb/datafusion-distributed/pull/57 (in particular) becomes much smaller. --- src/shm/README.md | 103 ++++++++++++++++ src/shm/dsm.rs | 8 +- src/shm/in_process.rs | 104 ++++++++++++---- src/shm/mod.rs | 14 ++- src/shm/mpsc_ring.rs | 28 ++++- src/shm/runtime.rs | 89 +++++++++++++- src/shm/self_hosted.rs | 133 +++++++++++--------- src/shm/setup.rs | 273 ++++++++++++++++++++++++++++++++++++----- src/shm/transport.rs | 201 ++++++++++++++++++++++++++++++ 9 files changed, 820 insertions(+), 133 deletions(-) create mode 100644 src/shm/README.md diff --git a/src/shm/README.md b/src/shm/README.md new file mode 100644 index 000000000..6993fcc40 --- /dev/null +++ b/src/shm/README.md @@ -0,0 +1,103 @@ +# Shared-Memory Transport (`src/shm`) + +This module provides a non-gRPC [`ChannelResolver`](../protocol/worker_channel.rs) and transport layer for co-located execution, where workers run as parallel tasks or processes sharing one machine and communicating over a shared-memory mesh rather than gRPC over TCP. + +--- + +## Architecture Overview + +The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC design while replacing network serialization and TCP sockets with Direct Shared Memory (DSM) ring buffers and compact 16-byte frame headers. + +```text +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Coordinator │ +└──────────────────────────────────────┬──────────────────────────────────────┘ + │ SetPlan (Control Mesh) + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Worker Process / Instance │ +│ │ +│ 1. Receives SetPlan & registers plan fragment (TaskKey) │ +│ 2. Idles until ExecuteTaskFrame arrives from downstream consumer │ +│ 3. ExecuteTaskFrame arrives specifying partition_range (start..end) │ +│ 4. Opens per-partition sinks lazily for start..end │ +│ 5. Evaluates run_worker_fragment(plan, sinks, ctx, start..end) │ +│ 6. Streams batches out via DSM ring buffers │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Control & Data Plane Design + +### 1. Control Plane (`MppMesh`, `DrainHandle` & `run_execute_task_loop`) +- **Messages**: Control frames (`ExecuteTask`, `SetPlan`, `TaskMetrics`, `Cancel`, `EOF`) are tagged with a 16-byte [`MppFrameHeader`](./transport.rs) (`kind`, `stage_id`, `partition`, `sender_proc`) and routed through `MppMesh`. +- **Demand-Driven Pull Execution**: + - Downstream consumers issue `ExecuteTaskFrame` to upstream producer tasks via `MppMesh::send_execute_task`. + - Upstream worker tasks wait on `MppMesh::take_execute_task_rx(stage_num, task_idx)` for incoming execution requests before spawning task fragments. + - Workers run the demand loop via [`run_execute_task_loop`](./setup.rs), which validates requested partition ranges (`start..end`), guards against duplicate/overlapping partition claims, listens to `CancellationToken` for prompt cancellation unwinding, and periodically drives inbound ring draining. +- **Demuxing**: Incoming frames are demuxed cooperatively by [`DrainHandle`](./transport.rs) into per-`(stage_id, task_number)` request registries and per-channel record batch buffers. + +### 2. Data Plane (DSM Ring Buffers) +- Output partitions write record batches into lock-free/cache-aligned DSM ring buffers (`mpsc_ring.rs`). +- Batches are serialized as Arrow IPC streams directly into ring slots. +- Dedicated per-partition rings eliminate head-of-line blocking across tasks. + +--- + +## Lifecycle & Memory Safety + +- **Session Handles (`LeaderSession` / `WorkerSession`)**: `leader_setup` and `worker_setup` hand back `LeaderSession` and `WorkerSession` handles owning the process's outbound data senders. +- **Scope Retention**: Internal sender fields are private to prevent premature destructuring at compile time. Embedders hold `session` as a local variable for the execution scope. +- **Teardown & Detachment**: When `session` drops (on normal return, `?` error, or panic/unwind), its outbound senders drop automatically, causing peer inboxes to observe `Detached` without requiring manual cleanup calls. Embedders invoke `MppMesh::mark_detached()` when unmapping shared-memory segments to clear liveness flags. + +--- + +## Embedder Usage & Session Lifecycles + +### Worker Execution Loop (`pg_search` / parallel workers) + +```rust +pub fn run_mpp_worker(...) -> Result<()> { + // 1. Attach to shared memory region and obtain the opaque WorkerSession handle. + let session = unsafe { + worker_setup(base, region_total, proc_idx, wakeup, token, interrupt)? + }; + + // 2. Access session.mesh and session.plan_bytes. + let ctx = build_session(Arc::clone(&session.mesh)); + + // 3. Drive worker execution loop. + run_execute_task_loop(&session.mesh, ...).await?; + + Ok(()) + // 4. `session` drops here on normal exit, `?` error, or panic unwind, + // automatically detaching peer inboxes. +} +``` + +### Leader Query Execution + +```rust +pub async fn run_leader_query(...) -> Result> { + // 1. Initialize DSM region and obtain LeaderSession. + let session = unsafe { + leader_setup(base, n_procs, queue_bytes, plan_bytes, wakeup, token, interrupt, true)? + }; + + // 2. Execute plan over session.mesh. + let results = execute_plan(&session.mesh).await?; + + Ok(results) + // 3. `session` drops at query completion, releasing leader control senders. +} +``` + +--- + +## Extension Points for Embedders + +Embedders (such as ParadeDB / Postgres backend engines or custom runners) consume the `src/shm` API: +- **Buffer Allocation**: Standard POSIX `mmap` or Postgres Shared Memory allocations (`dsm.rs`). +- **Worker Execution Loop**: [`run_execute_task_loop`](./setup.rs) for driving worker tasks, range validation, and cancellation unwinding. +- **Wakeup Primitives**: Custom signal/wakeup hooks (`NO_RECEIVER_TOKEN`, [`Wakeup`](./mpsc_ring.rs)). diff --git a/src/shm/dsm.rs b/src/shm/dsm.rs index 3720da0eb..6dca74aa5 100644 --- a/src/shm/dsm.rs +++ b/src/shm/dsm.rs @@ -383,8 +383,12 @@ unsafe fn attach_proc( let inbox_addr = unsafe { base.add(off) }; let nn = unsafe { mpsc_ring::attach_at(inbox_addr, ring_slots, slot_capacity) } .expect("DsmMpscRing attach_at: leader-initialized region must validate"); - outbound_senders - .push(unsafe { DsmMpscSender::new(nn, Arc::clone(&wakeup), Arc::clone(&alive)) }); + let sender = if this_proc == 0 { + unsafe { DsmMpscSender::new_control(nn, Arc::clone(&wakeup), Arc::clone(&alive)) } + } else { + unsafe { DsmMpscSender::new(nn, Arc::clone(&wakeup), Arc::clone(&alive)) } + }; + outbound_senders.push(sender); } } diff --git a/src/shm/in_process.rs b/src/shm/in_process.rs index 4de15adbf..f3fd0d20b 100644 --- a/src/shm/in_process.rs +++ b/src/shm/in_process.rs @@ -64,9 +64,7 @@ use crate::{ use super::mpsc_ring::Wakeup; use super::runtime::{InProcessWorkerResolver, MppMesh, ShmChannelResolver, proc_for_task}; -use super::setup::{ - collect_task_metrics, dsm_region_bytes, leader_setup, run_worker_fragment, worker_setup, -}; +use super::setup::{collect_task_metrics, dsm_region_bytes, leader_setup, worker_setup}; use super::transport::{ CooperativeDrainSet, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, }; @@ -148,6 +146,7 @@ enum FragmentRouting { /// One producer stage's routing metadata, captured from a network boundary. The stage's plan is /// not captured here: workers run the specialized plans the coordinator dispatches, delivered /// through the leader session's [`CapturingPlanSource`]. +#[derive(Clone, Debug)] struct StageEntry { stage_num: u32, task_count: usize, @@ -379,14 +378,22 @@ async fn run_worker_proc( for fragment in &fragments { routing.insert(fragment.stage_id, fragment.routing.clone()); } + // The metrics frames go to the leader after the fragments finish; the clone keeps one sender + // on the leader's inbox alive past the drop below, which only delays that ring's detach + // observation, never a per-channel EOF. + let metrics_sender_base = outbound + .first() + .and_then(|s| s.as_ref()) + .map(|s| s.clone_with_header(MppFrameHeader::task_metrics(0, 0, mesh.this_proc))); + // One sink serves every stage this proc produces; it owns the base outbound senders and routes // each (stage, partition) to the destination proc's send end. - let worker_sink = ShmMqWorkerSink { + let worker_sink = Arc::new(ShmMqWorkerSink { outbound, mesh: Arc::clone(&mesh), n_workers, routing, - }; + }); let mut prepared = Vec::with_capacity(fragments.len()); for fragment in &fragments { @@ -395,34 +402,71 @@ async fn run_worker_proc( let plan = reinstantiate(&captured_plan(&captured, fragment.stage_id, fragment.task_idx).await); let n_out = plan.output_partitioning().partition_count(); - let mut sinks: Vec> = Vec::with_capacity(n_out); - for q in 0..n_out { - sinks.push(worker_sink.open_partition(fragment.stage_id as usize, q)?); + prepared.push((fragment, plan, n_out, task_ctx)); + } + + struct AbortOnDrop(tokio::task::JoinHandle<()>); + impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); } - prepared.push((fragment, plan, sinks, task_ctx)); } - // The metrics frames go to the leader after the fragments finish; the clone keeps one sender - // on the leader's inbox alive past the drop below, which only delays that ring's detach - // observation, never a per-channel EOF. - let metrics_sender_base = worker_sink - .outbound - .first() - .and_then(|s| s.as_ref()) - .map(|s| s.clone_with_header(MppFrameHeader::task_metrics(0, 0, mesh.this_proc))); - // Drop the base senders so the only senders left are the per-partition clones the fragment - // futures own; otherwise the rings never observe the last-sender detach. - drop(worker_sink); + + let drain_mesh = Arc::clone(&mesh); + // `in_process.rs` is an in-process test harness running under a multi-threaded Tokio test + // runtime. Unlike production cooperative workers (which run single-threaded without tokio::spawn), + // spawning a background task here is acceptable to keep mock worker meshes drained during tests. + #[allow(clippy::disallowed_methods)] + let _drain_guard = AbortOnDrop(tokio::spawn(async move { + loop { + if drain_mesh.drain_all_inbound().is_err() { + break; + } + tokio::task::yield_now().await; + } + })); let mut futures = Vec::with_capacity(prepared.len()); let mut executed = Vec::with_capacity(prepared.len()); - for (fragment, plan, sinks, task_ctx) in prepared { + for (fragment, plan, n_out, task_ctx) in prepared { executed.push(( fragment.stage_id, fragment.task_idx, fragment.task_count, Arc::clone(&plan), )); - futures.push(run_worker_fragment(plan, sinks, task_ctx)); + let mesh = Arc::clone(&mesh); + let stage_id = fragment.stage_id; + let task_idx = fragment.task_idx as u32; + let worker_sink = Arc::clone(&worker_sink); + + futures.push(async move { + let rx = mesh.take_execute_task_rx(stage_id, task_idx)?; + let mesh = Arc::clone(&mesh); + crate::shm::setup::run_execute_task_loop( + rx, + n_out, + tokio_util::sync::CancellationToken::new(), + || { + mesh.inbound_receiver().try_drain_pass()?; + Ok(()) + }, + |_request, _headers, range| { + let plan = Arc::clone(&plan); + let task_ctx = Arc::clone(&task_ctx); + let worker_sink = Arc::clone(&worker_sink); + async move { + let mut task_sinks = Vec::with_capacity(range.len()); + for q in range.clone() { + task_sinks.push(worker_sink.open_partition(stage_id as usize, q)?); + } + crate::shm::setup::run_worker_fragment(plan, task_sinks, task_ctx, range) + .await + } + }, + ) + .await + }); } for r in futures::future::join_all(futures).await { r?; @@ -739,7 +783,7 @@ mod tests { Arc::clone(&wakeup), receiver_token(0), Arc::new(NoInterrupt), - /* attach_senders */ false, + /* attach_senders */ true, ) } .unwrap() @@ -757,7 +801,11 @@ mod tests { ) } .unwrap(); - worker_setups.push((proc_idx, attach.mesh, attach.outbound_senders)); + worker_setups.push(( + proc_idx, + Arc::clone(&attach.mesh), + attach.outbound_senders().to_vec(), + )); } // Build the distributed plan once on the leader session; producers and consumer share it. @@ -893,7 +941,7 @@ mod tests { Arc::clone(&wakeup), receiver_token(0), Arc::new(NoInterrupt), - /* attach_senders */ false, + /* attach_senders */ true, ) } .unwrap() @@ -911,7 +959,11 @@ mod tests { ) } .unwrap(); - workers.push((proc_idx, attach.mesh, attach.outbound_senders)); + workers.push(( + proc_idx, + Arc::clone(&attach.mesh), + attach.outbound_senders().to_vec(), + )); } Bootstrap { leader_mesh, diff --git a/src/shm/mod.rs b/src/shm/mod.rs index bf5c95e17..0c2e2f37f 100644 --- a/src/shm/mod.rs +++ b/src/shm/mod.rs @@ -23,11 +23,16 @@ //! drain) live here as a reusable library; an embedder supplies the two platform primitives via //! small extension points: how to allocate the shared buffer, and how to wake a blocked consumer. //! +//! The shared-memory transport implements a demand-driven, pull-based RPC model: downstream +//! consumers request partition execution on demand via [`ExecuteTaskFrame`], matching the +//! canonical gRPC task execution model over shared-memory channels. +//! //! The point of hosting it in this crate is testing: the in-process instantiation runs real //! distributed queries through the transport in this crate's CI, so an upstream rebase that //! breaks the channel-protocol contract fails here, before any downstream embedder rebuilds. //! //! [`ChannelResolver`]: crate::ChannelResolver +//! [`ExecuteTaskFrame`]: transport::ExecuteTaskFrame //! //! Two assumptions an embedder signs up for: //! - Execution is cooperative on a current-thread runtime: consumers spin on @@ -60,13 +65,14 @@ mod transport; pub use mpsc_ring::{NO_RECEIVER_TOKEN, Wakeup}; pub use runtime::{InProcessWorkerResolver, MppMesh, ShmChannelResolver, proc_for_task}; pub use setup::{ - LeaderAttach, WorkerAttach, collect_task_metrics, dsm_region_bytes, install_work_unit_channels, - leader_setup, region_total, run_worker_fragment, worker_setup, + LeaderSession, WorkerSession, collect_task_metrics, dsm_region_bytes, + install_work_unit_channels, leader_setup, region_total, run_execute_task_loop, + run_worker_fragment, worker_setup, }; pub use sink::{PartitionSink, WorkerSink}; pub use transport::{ - CooperativeDrainSet, Interrupt, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, - SendBatchStats, SetPlanFrame, + CooperativeDrainSet, ExecuteTaskFrame, ExecuteTaskRx, Interrupt, MppFrameHeader, + MppPartitionSink, MppSender, NoInterrupt, SendBatchStats, SetPlanFrame, }; /// Out-of-DSM liveness flag shared by the ring handles from one attach. The embedder flips it to diff --git a/src/shm/mpsc_ring.rs b/src/shm/mpsc_ring.rs index 93e13d7e2..4311eec76 100644 --- a/src/shm/mpsc_ring.rs +++ b/src/shm/mpsc_ring.rs @@ -464,12 +464,13 @@ impl DsmMpscReceiver { let seq = unsafe { (*slot).seq.load(Ordering::Acquire) }; let expected_ready = head.wrapping_add(1); if seq != expected_ready { - // Slot not ready. Use `<=` rather than `==` so a strict invariant - // violation (tail < head, impossible under correct operation) still - // surfaces as Detached rather than wedging Empty forever. - if header.detached.load(Ordering::Acquire) - && header.tail.load(Ordering::Acquire) <= head - { + // If the next expected slot is not yet ready, check if producers have detached. + // Under teardown or worker exit, once `detached` latches to true, no further slots + // will be published. Checking `detached` directly here ensures the consumer unblocks + // immediately with `RecvOutcome::Detached` during teardown, rather than remaining + // wedged on `RecvOutcome::Empty` waiting for a claimed-but-unpublished slot from an + // exited sender. + if header.detached.load(Ordering::Acquire) { return RecvOutcome::Detached; } return RecvOutcome::Empty; @@ -650,6 +651,21 @@ impl DsmMpscSender { } } + /// Wrap an already-initialized ring as a control sender. Does NOT increment `sender_count`, + /// so it doesn't prevent the consumer from detecting when all true data producers drop. + pub(super) unsafe fn new_control( + ring: NonNull, + wakeup: Arc, + alive: AliveFlag, + ) -> Self { + Self { + ring, + wakeup, + counts_as_data: false, + alive, + } + } + /// A control-plane sibling onto the same ring this handle already targets: it can publish /// frames but stays out of `sender_count`, so it never sets or delays `detached`. Used to derive /// a proc's `Cancel` senders from its data senders without a second attach, so a consumer can diff --git a/src/shm/runtime.rs b/src/shm/runtime.rs index 6b6169051..55bf6c49b 100644 --- a/src/shm/runtime.rs +++ b/src/shm/runtime.rs @@ -132,6 +132,7 @@ impl MppMesh { /// so handles dropped afterward (e.g. by a memory-context reset) never touch freed memory. pub fn mark_detached(&self) { self.alive.store(false, Ordering::Release); + *self.cancel_senders.lock().unwrap() = None; } /// The raw liveness flag shared with every ring handle, for embedders that register a C @@ -178,6 +179,47 @@ impl MppMesh { sender.send_set_plan_traced(&frame, &mut stats).await } + /// Sends an `ExecuteTask` control frame to `dest_proc` to request partition stream execution + /// for `(stage_id, task_number)`. + pub async fn send_execute_task( + self: &Arc, + stage_id: u32, + task_number: u32, + frame: crate::shm::transport::ExecuteTaskFrame, + ) -> Result<()> { + let dest_proc = proc_for_task(self.n_workers(), task_number); + let sender = { + let guard = self.cancel_senders.lock().unwrap(); + let Some(senders) = guard.as_ref() else { + return internal_err!( + "shm mesh: control senders not installed; cannot route ExecuteTask" + ); + }; + let senders = senders.lock().unwrap(); + let Some(Some(base)) = senders.get(dest_proc as usize) else { + return internal_err!("shm mesh: no control sender for proc {dest_proc}"); + }; + base.clone_with_header(crate::shm::transport::MppFrameHeader::execute_task( + stage_id, + task_number, + self.this_proc, + )) + .with_cooperative_drain(Arc::clone(self) as Arc) + }; + let mut stats = crate::shm::transport::SendBatchStats::default(); + sender.send_execute_task_traced(&frame, &mut stats).await + } + + /// Obtains the channel receiver for incoming `ExecuteTask` requests targeting `(stage_id, task_number)`. + pub fn take_execute_task_rx( + self: &Arc, + stage_id: u32, + task_number: u32, + ) -> Result { + self.inbound_receiver + .take_execute_task_rx(stage_id, task_number) + } + /// Tell the producer on `producer_proc` to stop the `(stage_id, partition)` stream: this proc's /// consumer of it stopped reading before EOF. Ships one `Cancel` frame, leaving the rings /// healthy for metrics and every other stream. A no-op when no senders are installed (the @@ -198,7 +240,7 @@ impl MppMesh { /// The single cooperative inbound handle that pulls frames from every peer (and the /// self-loop) into per-`(sender_proc, stage_id, partition)` channel buffers. - pub(super) fn inbound_receiver(&self) -> &Arc { + pub fn inbound_receiver(&self) -> &Arc { &self.inbound_receiver } @@ -371,7 +413,7 @@ impl WorkerChannel for ShmWorkerChannel { async fn execute_task( &mut self, - _headers: HeaderMap, + headers: HeaderMap, request: ExecuteTaskRequest, metrics: ExecutionPlanMetricsSet, _task_ctx: &Arc, @@ -403,6 +445,49 @@ impl WorkerChannel for ShmWorkerChannel { self.mesh.n_procs ))); } + + let pb_request = crate::proto::ExecuteTaskRequest { + task_key: Some(crate::proto::TaskKey { + query_id: crate::common::serialize_uuid(&request.task_key.query_id), + stage_id: request.task_key.stage_id as u64, + task_number: request.task_key.task_number as u64, + }), + target_partition_start: request.target_partition_start as u64, + target_partition_end: request.target_partition_end as u64, + producer_head: Some(match request.producer_head_spec { + crate::protocol::ProducerHeadSpec::None => { + crate::proto::execute_task_request::ProducerHead::None( + crate::proto::NoneHead {}, + ) + } + crate::protocol::ProducerHeadSpec::BroadcastExec { output_partitions } => { + crate::proto::execute_task_request::ProducerHead::Broadcast( + crate::proto::BroadcastExecHead { + output_partitions: output_partitions as u64, + }, + ) + } + crate::protocol::ProducerHeadSpec::RepartitionExec { partitioning } => { + crate::proto::execute_task_request::ProducerHead::Repartition( + crate::proto::RepartitionExecHead { partitioning }, + ) + } + }), + }; + + let frame = match crate::shm::transport::ExecuteTaskFrame::from_parts(pb_request, &headers) + { + Ok(f) => f, + Err(e) => { + return Err(DataFusionError::Internal(format!( + "ExecuteTask frame build failed: {e}" + ))); + } + }; + self.mesh + .send_execute_task(stage_id, task_number, frame) + .await?; + // First line to grep when a query hangs on a channel nothing registered. log::debug!( "shm transport execute_task: this_proc={} stage_id={stage_id} \ diff --git a/src/shm/self_hosted.rs b/src/shm/self_hosted.rs index a14b00378..db0f473fb 100644 --- a/src/shm/self_hosted.rs +++ b/src/shm/self_hosted.rs @@ -720,7 +720,7 @@ impl QueryHarness { Arc::clone(&interrupt), ) }?; - worker_meshes.push((attach.mesh, attach.outbound_senders)); + worker_meshes.push((attach.mesh, attach.outbound_senders().to_vec())); } // Build every fragment's launch package: one routed sink per output partition, sharing @@ -784,7 +784,7 @@ impl QueryHarness { state.n_workers = n_workers; state.leader_mesh = Some(leader_mesh); - state.leader_senders = leader_attach.outbound_senders; + state.leader_senders = leader_attach.outbound_senders().to_vec(); state.worker_meshes = worker_meshes.iter().map(|(m, _)| Arc::clone(m)).collect(); state.launches = launches; state.region = Some(region); @@ -893,67 +893,80 @@ impl TaskDriver { ); } - let produce = async { - let request = pb::ExecuteTaskRequest { - task_key: Some(task_key), - target_partition_start: 0, - target_partition_end: n_partitions as u64, - producer_head: Some(producer_head), - }; - // Through `execute_local_task` rather than a bare `plan.execute` so the task metrics - // (added/executed/finished stamps, per-node metrics) flow exactly like a pull-based - // read would deliver them. - let (streams, _ctx) = execute_local_task(worker.task_data_entries(), request).await?; - if streams.len() != launch.sinks.len() { - return internal_err!( - "self-hosted shm transport: stage {stage_num} task {task_i} decoded into {} \ - partitions but routed {} sinks", - streams.len(), - launch.sinks.len() - ); - } - let mut futures = Vec::with_capacity(streams.len()); - for (mut stream, mut sink) in streams.into_iter().zip(launch.sinks) { + let rx = launch.mesh.take_execute_task_rx(stage_num, task_i as u32)?; + let mut sinks_opt: Vec<_> = launch.sinks.into_iter().map(Some).collect(); + let mesh = Arc::clone(&launch.mesh); + let produce = crate::shm::setup::run_execute_task_loop( + rx, + n_partitions, + token.clone(), + || { + mesh.inbound_receiver().try_drain_pass()?; + Ok(()) + }, + |request, _headers, range| { + let len = range.len(); + let mut task_sinks = Vec::with_capacity(len); + for i in range { + task_sinks.push(sinks_opt[i].take().unwrap()); + } let token = token.clone(); - futures.push(async move { - let stream_result: Result<()> = async { - loop { - let batch = tokio::select! { - next = stream.next() => next, - // Head stream dropped: stop pulling so this fragment and its upstream - // scan unwind, instead of draining the input into a buffer no one reads. - _ = token.cancelled() => break, - }; - let Some(batch) = batch else { break }; - let batch = batch?; - if batch.num_rows() == 0 { - continue; - } - sink.send(&batch).await?; - // A downstream worker abandoned this stream (its mesh carries the cancel - // senders): stop pulling so the cancel cascades to this fragment's own - // producers, matching the embedder's `run_worker_fragment` loop. - if sink.cancelled() { - break; + let worker = Arc::clone(&worker); + async move { + // execute_local_task gives streams for the requested range. + let (streams, _ctx) = + execute_local_task(worker.task_data_entries(), request).await?; + if streams.len() != task_sinks.len() { + return internal_err!( + "self-hosted shm transport: stage {stage_num} task {task_i} decoded into {} \ + partitions but routed {} sinks", + streams.len(), + task_sinks.len() + ); + } + + let mut stream_futs = Vec::with_capacity(streams.len()); + for (mut stream, mut sink) in streams.into_iter().zip(task_sinks) { + let token = token.clone(); + stream_futs.push(async move { + let stream_result: Result<()> = async { + loop { + let batch = tokio::select! { + next = stream.next() => next, + // Head stream dropped: stop pulling so this fragment and its upstream + // scan unwind, instead of draining the input into a buffer no one reads. + _ = token.cancelled() => break, + }; + let Some(batch) = batch else { break }; + let batch = batch?; + if batch.num_rows() == 0 { + continue; + } + sink.send(&batch).await?; + // A downstream worker abandoned this stream (its mesh carries the cancel + // senders): stop pulling so the cancel cascades to this fragment's own + // producers, matching the embedder's `run_worker_fragment` loop. + if sink.cancelled() { + break; + } + } + Ok(()) } - } - Ok(()) + .await; + // EOF always, even after a failed send, so the consumer side unblocks; the + // stream error stays the primary one. + let eof_result = sink.finish().await; + stream_result.and(eof_result) + }); } - .await; - // EOF always, even after a failed send, so the consumer side unblocks; the - // stream error stays the primary one. - let eof_result = sink.finish().await; - stream_result.and(eof_result) - }); - } - // `join_all`, not fail-fast: cancelling sibling partitions mid-await would skip their - // EOFs and wedge the consumer's channel buffers. - let results = futures::future::join_all(futures).await; - for r in results { - r?; - } - Ok(()) - }; + let results = futures::future::join_all(stream_futs).await; + for r in results { + r?; + } + Ok(()) + } + }, + ); let produce_res: Result<()> = produce.await; // The metrics receiver resolves as the last partition stream above completes, so this diff --git a/src/shm/setup.rs b/src/shm/setup.rs index d630c63cb..21a59a029 100644 --- a/src/shm/setup.rs +++ b/src/shm/setup.rs @@ -29,12 +29,14 @@ //! pure Rust over the shared buffer. use std::ffi::c_void; +use std::future::Future; use std::sync::Arc; use datafusion::common::{DataFusionError, Result}; use datafusion::execution::TaskContext; -use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion::physical_plan::ExecutionPlan; use futures::stream::StreamExt; +use tokio_util::sync::CancellationToken; use super::dsm::{ compute_dsm_layout, leader_init, peer_proc_for_index, read_region_total, worker_attach, @@ -43,8 +45,8 @@ use super::mesh::{DsmInboxReceiver, DsmInboxSender}; use super::mpsc_ring::{DsmMpscSender, NO_RECEIVER_TOKEN, Wakeup}; use super::runtime::MppMesh; use super::transport::{ - BatchChannelSender, DrainHandle, Interrupt, MppFrameHeader, MppReceiver, MppSender, - ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, + BatchChannelSender, DrainHandle, ExecuteTaskRx, Interrupt, MppFrameHeader, MppReceiver, + MppSender, ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, }; use crate::proto as pb; use crate::work_unit_feed::RemoteWorkUnitFeedRegistry; @@ -118,19 +120,20 @@ fn build_outbound_senders( (senders, cancel) } -/// What [`leader_setup`] hands back to the embedder. -pub struct LeaderAttach { +/// What [`leader_setup`] hands back to the embedder: the active leader session on the mesh. +/// +/// Dropping this handle drops `_outbound_senders`, notifying peer worker inboxes that the leader +/// has detached. Fields `mesh` and `plan_bytes` are accessible, while `_outbound_senders` is private +/// to prevent premature destructuring. +pub struct LeaderSession { /// The leader's mesh, installed on its DataFusion session. pub mesh: Arc, /// Outbound senders keyed by destination proc index, for the control plane: work-unit - /// frames flow leader -> worker through them. Slot 0 (the leader itself) stays `None`; - /// empty unless `attach_senders` was passed. Holders must keep them alive for the whole - /// query: dropping them before a worker attaches latches that worker's inbox as detached. - pub outbound_senders: Vec>, + /// frames flow leader -> worker through them. Private to enforce session scope retention. + _outbound_senders: Vec>, } -/// Initialize the shared region as the leader (`proc 0`) and return its mesh plus its outbound -/// senders. +/// Initialize the shared region as the leader (`proc 0`) and return its session handle. /// /// Writes the region header, copies `plan_bytes` in, initializes the `n_procs` inboxes, and /// attaches the leader as receiver to its own inbox. `receiver_token` is registered so producers @@ -151,7 +154,7 @@ pub unsafe fn leader_setup( receiver_token: u64, interrupt: Arc, attach_senders: bool, -) -> Result { +) -> Result { if receiver_token == NO_RECEIVER_TOKEN { return Err(DataFusionError::Internal( "mpp: leader_setup: receiver_token is the NO_RECEIVER_TOKEN sentinel; wakeups \ @@ -182,13 +185,13 @@ pub unsafe fn leader_setup( // work-unit frames (and later dynamic filters) flow leader -> worker through them. Empty // when the embedder did not opt in: a ring latches `detached` once its sender count hits // zero, so senders that might drop before every worker attached must never exist. - // The leader's `Cancel` senders are wired by the embedder (it shares the same outbound senders - // it holds for plan delivery and releases before the DSM unmaps), so drop the cancel set here. - let (outbound_senders, _cancel_senders) = + let (outbound_senders, cancel_senders) = build_outbound_senders(0, n_procs, attach.outbound_senders); - Ok(LeaderAttach { - mesh: Arc::new(MppMesh::new(0, n_procs, inbound, interrupt, attach.alive)), - outbound_senders, + let mesh = Arc::new(MppMesh::new(0, n_procs, inbound, interrupt, attach.alive)); + mesh.set_cancel_senders(Arc::new(std::sync::Mutex::new(cancel_senders))); + Ok(LeaderSession { + mesh, + _outbound_senders: outbound_senders, }) } @@ -241,15 +244,29 @@ pub fn install_work_unit_channels( mesh.register_work_unit_senders(stage_id, task_number, channels.senders); } -/// What [`worker_setup`] hands back to the embedder. -pub struct WorkerAttach { +/// What [`worker_setup`] hands back to the embedder: the active worker session on the mesh. +/// +/// Dropping this handle drops `_outbound_senders`, notifying peer inboxes that this worker +/// has detached. Private `_outbound_senders` prevents premature destructuring. +pub struct WorkerSession { /// The worker's mesh, installed on its DataFusion session. pub mesh: Arc, - /// Outbound senders keyed by destination proc index. The slot at `this_proc` is the in-proc - /// self-loop; every other slot writes to that peer's inbox. - pub outbound_senders: Vec>, /// The plan bytes the leader wrote into the region, copied out for this worker. pub plan_bytes: Vec, + /// Outbound senders keyed by destination proc index. Private to enforce session scope retention. + _outbound_senders: Vec>, +} + +impl LeaderSession { + pub fn outbound_senders(&self) -> &[Option] { + &self._outbound_senders + } +} + +impl WorkerSession { + pub fn outbound_senders(&self) -> &[Option] { + &self._outbound_senders + } } /// Attach to the leader-initialized region as worker `proc_idx` (`>= 1`). @@ -264,7 +281,7 @@ pub unsafe fn worker_setup( wakeup: Arc, receiver_token: u64, interrupt: Arc, -) -> Result { +) -> Result { if receiver_token == NO_RECEIVER_TOKEN { return Err(DataFusionError::Internal( "mpp: worker_setup: receiver_token is the NO_RECEIVER_TOKEN sentinel; wakeups \ @@ -277,7 +294,7 @@ pub unsafe fn worker_setup( .map_err(DataFusionError::Internal)?; let total_procs = header.n_procs; - let (mut outbound, cancel) = + let (mut outbound, mut cancel) = build_outbound_senders(proc_idx, total_procs, attach.outbound_senders); // Self-loop in-proc channel: peer-mesh routing can land a producer and its consumer on the same @@ -286,7 +303,11 @@ pub unsafe fn worker_setup( let (self_tx, self_rx) = in_proc_channel(SELF_LOOP_CAPACITY); let self_tx_arc: Arc = Arc::new(self_tx); outbound[proc_idx as usize] = Some(MppSender::with_header( - self_tx_arc, + Arc::clone(&self_tx_arc), + MppFrameHeader::batch(0, 0, proc_idx), + )); + cancel[proc_idx as usize] = Some(MppSender::with_header( + Arc::clone(&self_tx_arc), MppFrameHeader::batch(0, 0, proc_idx), )); @@ -310,10 +331,10 @@ pub unsafe fn worker_setup( // its mesh the control-plane cancel senders; they drop with the mesh at the end of the worker's // run, well before the DSM unmaps, so no explicit release is needed. mesh.set_cancel_senders(Arc::new(std::sync::Mutex::new(cancel))); - Ok(WorkerAttach { + Ok(WorkerSession { mesh, - outbound_senders: outbound, plan_bytes, + _outbound_senders: outbound, }) } @@ -328,16 +349,19 @@ pub async fn run_worker_fragment( plan: Arc, sinks: Vec>, ctx: Arc, + partition_range: std::ops::Range, ) -> Result<()> { - let n_partitions = plan.output_partitioning().partition_count(); - if n_partitions != sinks.len() { + if partition_range.end.saturating_sub(partition_range.start) != sinks.len() { return Err(DataFusionError::Internal(format!( - "run_worker_fragment: plan has {n_partitions} output partitions but {} sinks", + "run_worker_fragment: partition range {}..{} requires {} sinks but got {}", + partition_range.start, + partition_range.end, + partition_range.end.saturating_sub(partition_range.start), sinks.len() ))); } - let mut futures = Vec::with_capacity(n_partitions); - for (partition, mut sink) in sinks.into_iter().enumerate() { + let mut futures = Vec::with_capacity(sinks.len()); + for (partition, mut sink) in partition_range.zip(sinks.into_iter()) { let plan = Arc::clone(&plan); let ctx = Arc::clone(&ctx); futures.push(async move { @@ -371,3 +395,186 @@ pub async fn run_worker_fragment( } Ok(()) } + +/// Run the worker request loop for receiving [`ExecuteTaskFrame`] requests, validating partition +/// ranges, and delegating execution of each requested range to `spawn_range`. +/// +/// # Lifecycle & Guarantees +/// - **On-Demand Range Execution:** Listens on `rx` for incoming [`ExecuteTaskFrame`] messages, +/// extracting the requested partition range (`start..end`) and headers. +/// - **Partition Validation & Single-Claim Accounting:** Ensures that `start <= end` and `end <= n_partitions`, +/// and verifies that no partition within `start..end` has been claimed by a previous frame. Returns an +/// [`DataFusionError::Execution`] error if bounds are exceeded or duplicate partition claims occur. +/// - **Cancellation & Early Termination Unwinding:** Selects on `token.cancelled()`. If the query is +/// cancelled or terminates early (e.g. satisfied by a `LIMIT` clause downstream), the request loop +/// breaks out immediately and drops active sub-futures. +/// - **Cooperative Inbound Ring Flushing:** Periodically invokes `drain_pass` to drive the inbound ring +/// buffer drain pass while awaiting frame arrivals or stream completions. +/// - **Completion:** Exits cleanly once all `n_partitions` have been requested (or `rx` closes) AND all +/// spawned stream futures in `spawn_range` have completed to EOF. +/// +/// # Arguments +/// * `rx`: The receiver channel for incoming [`ExecuteTaskFrame`] messages obtained via `MppMesh::take_execute_task_rx`. +/// * `n_partitions`: Total number of output partitions expected across all request frames for this task. +/// * `token`: Cancellation token used to unwind the request loop on query cancellation or early teardown. +/// * `drain_pass`: Callback invoked periodically to drive inbound ring buffer draining (e.g. `mesh.inbound_receiver().try_drain_pass()`). +/// * `spawn_range`: Closure invoked for each valid, unclaimed partition range `start..end`. Must return a future executing the partition streams for that range. +pub async fn run_execute_task_loop( + rx: ExecuteTaskRx, + n_partitions: usize, + token: CancellationToken, + mut drain_pass: impl FnMut() -> Result<()>, + mut spawn_range: F, +) -> Result<()> +where + F: FnMut(pb::ExecuteTaskRequest, http::HeaderMap, std::ops::Range) -> Fut, + Fut: Future> + Send + 'static, +{ + if n_partitions == 0 { + return Ok(()); + } + + let mut sub_futures = futures::stream::FuturesUnordered::new(); + let mut partitions_requested = 0; + let mut rx_opt = Some(rx); + let mut claimed = vec![false; n_partitions]; + + loop { + // Stop when frame receiving has finished (all partitions requested or channel closed) + // and all active partition stream futures have completed. + if rx_opt.is_none() && sub_futures.is_empty() { + break; + } + + tokio::select! { + // Unwind immediately on early query cancellation or completion. + _ = token.cancelled() => break, + + frame_res = async { rx_opt.as_mut().unwrap().recv().await }, if rx_opt.is_some() => { + match frame_res { + Some(Ok(frame)) => { + let (request, headers) = frame.into_parts()?; + let start = request.target_partition_start as usize; + let end = request.target_partition_end as usize; + + if start > end || end > n_partitions { + return Err(DataFusionError::Execution(format!( + "shm transport: invalid partition range {start}..{end} for total partitions {n_partitions}" + ))); + } + + for (i, claimed_slot) in claimed[start..end].iter_mut().enumerate() { + if *claimed_slot { + let q = start + i; + return Err(DataFusionError::Execution(format!( + "shm transport: requested partition range {start}..{end} includes already claimed partition {q}" + ))); + } + *claimed_slot = true; + } + + let len = end - start; + let range_fut = spawn_range(request, headers, start..end); + sub_futures.push(range_fut); + + partitions_requested += len; + if partitions_requested >= n_partitions { + rx_opt = None; + } + } + Some(Err(e)) => return Err(e), + None => { + rx_opt = None; + } + } + } + next_res = sub_futures.next(), if !sub_futures.is_empty() => { + match next_res { + Some(Ok(())) => {} + Some(Err(e)) => return Err(e), + None => unreachable!(), + } + } + _ = tokio::time::sleep(std::time::Duration::from_millis(1)) => { + drain_pass()?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::transport::ExecuteTaskFrame; + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn run_execute_task_loop_multi_frame_sub_ranges() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let token = CancellationToken::new(); + let total_partitions = 4; + let ranges_processed = Arc::new(AtomicUsize::new(0)); + let total_partition_len_processed = Arc::new(AtomicUsize::new(0)); + + // Frame 1: sub-range 0..2 + let frame1 = ExecuteTaskFrame::from_parts( + pb::ExecuteTaskRequest { + task_key: Some(pb::TaskKey { + query_id: vec![], + stage_id: 1, + task_number: 0, + }), + target_partition_start: 0, + target_partition_end: 2, + producer_head: None, + }, + &http::HeaderMap::new(), + ) + .unwrap(); + + // Frame 2: sub-range 2..4 + let frame2 = ExecuteTaskFrame::from_parts( + pb::ExecuteTaskRequest { + task_key: Some(pb::TaskKey { + query_id: vec![], + stage_id: 1, + task_number: 0, + }), + target_partition_start: 2, + target_partition_end: 4, + producer_head: None, + }, + &http::HeaderMap::new(), + ) + .unwrap(); + + tx.send(Ok(frame1)).unwrap(); + tx.send(Ok(frame2)).unwrap(); + + let ranges_counter = Arc::clone(&ranges_processed); + let len_counter = Arc::clone(&total_partition_len_processed); + + let res = run_execute_task_loop( + rx, + total_partitions, + token, + || Ok(()), + move |_request, _headers, range| { + let r_counter = Arc::clone(&ranges_counter); + let l_counter = Arc::clone(&len_counter); + let len = range.len(); + async move { + r_counter.fetch_add(1, Ordering::SeqCst); + l_counter.fetch_add(len, Ordering::SeqCst); + Ok(()) + } + }, + ) + .await; + + assert!(res.is_ok()); + assert_eq!(ranges_processed.load(Ordering::SeqCst), 2); + assert_eq!(total_partition_len_processed.load(Ordering::SeqCst), 4); + } +} diff --git a/src/shm/transport.rs b/src/shm/transport.rs index a9a60d99f..2604f6ad3 100644 --- a/src/shm/transport.rs +++ b/src/shm/transport.rs @@ -92,6 +92,7 @@ pub(super) enum MppFrameKind { /// frame once `total_len` bytes have arrived, so no single ring-resident message ever /// exceeds the ring and frame size is never an error. Chunk = 7, + ExecuteTask = 8, } /// Payload of a `SetPlan` frame: the plan-delivery message a worker needs to run one task, @@ -154,6 +155,67 @@ impl SetPlanFrame { } } +/// Payload of an `ExecuteTask` frame: the request a consumer uses to ask a producer +/// to evaluate a specific partition range. +/// Control frame wrapping a [`pb::ExecuteTaskRequest`] and pass-through HTTP headers sent over +/// the shared-memory control mesh. +#[derive(Clone, PartialEq, prost::Message)] +pub struct ExecuteTaskFrame { + #[prost(message, optional, tag = "1")] + pub request: Option, + #[prost(string, repeated, tag = "2")] + pub header_keys: Vec, + #[prost(string, repeated, tag = "3")] + pub header_values: Vec, +} + +impl ExecuteTaskFrame { + pub fn from_parts( + request: pb::ExecuteTaskRequest, + headers: &http::HeaderMap, + ) -> Result { + let mut header_keys = Vec::with_capacity(headers.len()); + let mut header_values = Vec::with_capacity(headers.len()); + for (name, value) in headers { + let value = value.to_str().map_err(|e| { + DataFusionError::Internal(format!( + "mpp: non-ASCII header {name} cannot travel in a ExecuteTask frame: {e}" + )) + })?; + header_keys.push(name.as_str().to_string()); + header_values.push(value.to_string()); + } + Ok(Self { + request: Some(request), + header_keys, + header_values, + }) + } + + pub fn into_parts(self) -> Result<(pb::ExecuteTaskRequest, http::HeaderMap), DataFusionError> { + let request = self.request.ok_or_else(|| { + DataFusionError::Internal( + "mpp: ExecuteTask frame carries no ExecuteTaskRequest".to_string(), + ) + })?; + let mut headers = http::HeaderMap::with_capacity(self.header_keys.len()); + for (key, value) in self.header_keys.iter().zip(self.header_values.iter()) { + let name = http::header::HeaderName::from_bytes(key.as_bytes()).map_err(|e| { + DataFusionError::Internal(format!( + "mpp: ExecuteTask frame header name {key:?}: {e}" + )) + })?; + let value = http::header::HeaderValue::from_str(value).map_err(|e| { + DataFusionError::Internal(format!( + "mpp: ExecuteTask frame header value for {key}: {e}" + )) + })?; + headers.append(name, value); + } + Ok((request, headers)) + } +} + /// 16-byte prefix on every transport frame. /// /// The fixed layout `[magic, flags, stage_id, partition]` (4×u32) is what @@ -271,6 +333,16 @@ impl MppFrameHeader { } } + /// Build an `ExecuteTask` header for `(stage_id, task_number)`. + pub fn execute_task(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::ExecuteTask, sender_proc), + stage_id, + partition: task_number, + } + } + /// Build a `Cancel` header for the `(stage_id, partition)` stream, stamped with the consumer's /// `sender_proc`. Carries no payload; the producer reads it as "stop sending this stream." pub fn cancel(stage_id: u32, partition: u32, sender_proc: u32) -> Self { @@ -321,6 +393,7 @@ impl MppFrameHeader { 5 => Ok(MppFrameKind::SetPlan), 6 => Ok(MppFrameKind::Cancel), 7 => Ok(MppFrameKind::Chunk), + 8 => Ok(MppFrameKind::ExecuteTask), other => Err(DataFusionError::Internal(format!( "mpp: unknown frame kind {other:#x}" ))), @@ -527,6 +600,7 @@ enum FrameBody { FeedEof, TaskMetrics(pb::TaskMetrics), SetPlan(SetPlanFrame), + ExecuteTask(ExecuteTaskFrame), Cancel, } @@ -565,6 +639,11 @@ fn decode_frame(bytes: &[u8]) -> Result<(MppFrameHeader, FrameBody), DataFusionE .map_err(|e| DataFusionError::Internal(format!("mpp: set-plan decode: {e}")))?; Ok((header, FrameBody::SetPlan(frame))) } + MppFrameKind::ExecuteTask => { + let frame = ExecuteTaskFrame::decode(payload) + .map_err(|e| DataFusionError::Internal(format!("mpp: execute-task decode: {e}")))?; + Ok((header, FrameBody::ExecuteTask(frame))) + } MppFrameKind::Batch => { let mut reader = StreamReader::try_new(payload, None)?; let batch = reader.next().ok_or_else(|| { @@ -1178,6 +1257,22 @@ impl MppSender { result } + /// Ship a task's execute-task request as an `ExecuteTask` frame. + pub async fn send_execute_task_traced( + &self, + frame: &ExecuteTaskFrame, + stats: &mut SendBatchStats, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + encode_prost_frame_into(self.header, frame, &mut scratch)?; + self.spin_send_scratch(&scratch, stats).await + } + .await; + self.scratch.replace(scratch); + result + } + /// Close the feed channels of the task this sender's header names: the wire stand-in for /// Flight closing its coordinator stream, after which the worker-side feed streams end. pub async fn send_feed_eof_traced( @@ -1233,6 +1328,12 @@ impl MppSender { } } +impl Clone for MppSender { + fn clone(&self) -> Self { + self.clone_with_header(self.header) + } +} + /// Per-call timing + spin metrics for [`MppSender::send_batch_traced`]. /// All fields accumulate; callers zero or reuse as needed. #[derive(Default, Debug, Clone)] @@ -1342,6 +1443,9 @@ impl MppReceiver { Ok((header, FrameBody::SetPlan(frame))) => { RecvBatchOutcome::SetPlan { header, frame } } + Ok((header, FrameBody::ExecuteTask(frame))) => { + RecvBatchOutcome::ExecuteTask { header, frame } + } Ok((header, FrameBody::Cancel)) => RecvBatchOutcome::Cancel { header }, Err(e) => RecvBatchOutcome::Error(e), }; @@ -1451,6 +1555,11 @@ pub(super) enum RecvBatchOutcome { header: MppFrameHeader, frame: SetPlanFrame, }, + /// A task execution request named by `header.(stage_id, partition=task)`. + ExecuteTask { + header: MppFrameHeader, + frame: ExecuteTaskFrame, + }, /// The collected metrics of the task named by the header. TaskMetrics { header: MppFrameHeader, @@ -1538,6 +1647,8 @@ pub struct DrainHandle { /// pending-or-waiting shape as the feed registry: a frame arriving before the task asks /// buffers in `Pending`; a task asking first parks a oneshot the drain fulfills. set_plan_registry: Mutex, + /// Worker-side destination of `ExecuteTask` frames, keyed `(stage_id, task_number)`. + execute_task_registry: Mutex, /// `(stage_id, partition)` streams this proc's consumers abandoned, learned from inbound /// `Cancel` frames. A producer blocked on a full outbound checks it in its send spin and ends /// that stream cleanly, so a consumer that stopped reading early doesn't leave it spinning to @@ -1607,6 +1718,20 @@ enum SetPlanSlot { Waiting(tokio::sync::oneshot::Sender>), } +pub type ExecuteTaskRx = + tokio::sync::mpsc::UnboundedReceiver>; + +#[derive(Default)] +struct ExecuteTaskRegistry { + map: HashMap<(u32, u32), ExecuteTaskSlot>, + dead: Option, +} + +enum ExecuteTaskSlot { + Pending(Vec>), + Active(tokio::sync::mpsc::UnboundedSender>), +} + impl DrainHandle { /// Construct a cooperative drain handle. Channel buffers are populated lazily by /// [`Self::try_drain_pass`] when a frame arrives, or up-front by [`Self::register_channel`] @@ -1625,6 +1750,7 @@ impl DrainHandle { task_metrics_tx, task_metrics_rx: Mutex::new(Some(task_metrics_rx)), set_plan_registry: Mutex::new(SetPlanRegistry::default()), + execute_task_registry: Mutex::new(ExecuteTaskRegistry::default()), cancelled_streams: Mutex::new(HashSet::default()), } } @@ -1835,9 +1961,70 @@ impl DrainHandle { let _ = tx.send(Err(DataFusionError::Execution(reason.to_string()))); } } + let mut execs = self.execute_task_registry.lock().unwrap(); + execs.dead = Some(reason.to_string()); + for (_, slot) in execs.map.drain() { + if let ExecuteTaskSlot::Active(tx) = slot { + let _ = tx.send(Err(DataFusionError::Execution(reason.to_string()))); + } + } } } + pub(super) fn route_execute_task( + &self, + stage_id: u32, + task_number: u32, + frame: ExecuteTaskFrame, + ) { + let mut guard = self.execute_task_registry.lock().unwrap(); + if guard.dead.is_some() { + return; + } + let slot = guard + .map + .entry((stage_id, task_number)) + .or_insert_with(|| ExecuteTaskSlot::Pending(Vec::new())); + match slot { + ExecuteTaskSlot::Pending(queue) => queue.push(Ok(frame)), + ExecuteTaskSlot::Active(tx) => { + let _ = tx.send(Ok(frame)); + } + } + } + + pub(super) fn take_execute_task_rx( + &self, + stage_id: u32, + task_number: u32, + ) -> Result { + let mut guard = self.execute_task_registry.lock().unwrap(); + if let Some(msg) = &guard.dead { + return Err(DataFusionError::Execution(msg.clone())); + } + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + if let Some(slot) = guard.map.get_mut(&(stage_id, task_number)) { + match slot { + ExecuteTaskSlot::Active(_) => { + return Err(DataFusionError::Execution(format!( + "take_execute_task_rx: execute_task_rx for stage {stage_id} task {task_number} already taken" + ))); + } + ExecuteTaskSlot::Pending(queue) => { + for item in queue.drain(..) { + let _ = tx.send(item); + } + *slot = ExecuteTaskSlot::Active(tx); + } + } + } else { + guard + .map + .insert((stage_id, task_number), ExecuteTaskSlot::Active(tx)); + } + Ok(rx) + } + /// Register (or look up) the channel buffer for `(sender_proc, stage_id, partition)`. /// The returned `Arc` is the canonical destination for frames matching /// that key: `try_drain_pass` pushes into the same entry on every `Batch { header, .. }` @@ -1968,6 +2155,9 @@ impl DrainHandle { RecvBatchOutcome::SetPlan { header, frame } => { self.route_set_plan(header.stage_id, header.partition, frame); } + RecvBatchOutcome::ExecuteTask { header, frame } => { + self.route_execute_task(header.stage_id, header.partition, frame); + } RecvBatchOutcome::Cancel { header } => { self.note_cancel(header.stage_id, header.partition); } @@ -2107,6 +2297,16 @@ mod tests { use std::thread::JoinHandle; + #[test] + fn take_execute_task_rx_second_take_errors() { + let drain = DrainHandle::cooperative(0, vec![]); + let rx1 = drain.take_execute_task_rx(1, 0); + assert!(rx1.is_ok()); + let rx2 = drain.take_execute_task_rx(1, 0); + assert!(rx2.is_err()); + assert!(rx2.unwrap_err().to_string().contains("already taken")); + } + impl DrainBuffer { /// Block until a batch is available, EOF is reached, or the buffer is cancelled. /// @@ -2268,6 +2468,7 @@ mod tests { | RecvBatchOutcome::FeedEof { .. } | RecvBatchOutcome::TaskMetrics { .. } | RecvBatchOutcome::SetPlan { .. } + | RecvBatchOutcome::ExecuteTask { .. } | RecvBatchOutcome::Cancel { .. } => {} RecvBatchOutcome::Empty => {} RecvBatchOutcome::Detached => { From 04d7e114e22e6bd436a341885c9fc5bc90330f75 Mon Sep 17 00:00:00 2001 From: Moe Date: Fri, 7 Aug 2026 21:15:13 -0700 Subject: [PATCH 11/15] fix: scale range partitioning to an unknown layout at network boundaries (#67) # Ticket(s) Closed - N/A ## What This PR scales a `Partitioning::Range` layout to `UnknownPartitioning` when a network boundary scales a stage's properties. ## Why The old arm was a `debug_assert!` plus an unscaled clone. Release builds kept the clone, so `NetworkCoalesceExec` scaled every other layout but not ranges, and the consumer requested a partition set the producers never served. Both sides then waited on each other for the life of the query. That is the six-hour silent hang in the paradedb/paradedb#5892 benchmark runs: with `partition_by` on join keys, a co-partitioned join under a bare `LIMIT` reports its range layout up to the gather, and nothing reshuffles in between. ## How A task-scaled range layout has no representation: the consumer side of a coalesce boundary sees every input task's ranges repeated, and `RangePartitioning` cannot express repeated split points. The boundary math and the coalesce consumer run on counts alone, so an unknown layout with the scaled count behaves exactly like the flat layouts, and the rows never move. The range property is lost above the boundary; #68 tracks carrying it across so consumer-side joins can stay co-partitioned. Earlier revisions of this PR failed loudly instead, then broke the layout with a repartition under the gather. Both are unnecessary once the boundary scales the count correctly. ## Tests - Unit test: a range layout scales to `UnknownPartitioning` with the multiplied count. - The paradedb/paradedb#5894 regress test (`partition_by` indexes, a 3-way co-partitioned join, a bare `LIMIT 5`) passes against this rev. - `cargo test --lib` green in both feature configs; CI clippy clean. --- src/execution_plans/common.rs | 43 +++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/execution_plans/common.rs b/src/execution_plans/common.rs index 8fb302d63..4578c7e33 100644 --- a/src/execution_plans/common.rs +++ b/src/execution_plans/common.rs @@ -22,15 +22,38 @@ pub(super) fn scale_partitioning( Partitioning::RoundRobinBatch(p) => Partitioning::RoundRobinBatch(f(*p)), Partitioning::Hash(hash, p) => Partitioning::Hash(hash.clone(), f(*p)), Partitioning::UnknownPartitioning(p) => Partitioning::UnknownPartitioning(f(*p)), - // A range shuffle never reaches a network boundary, so its count is never - // scaled. If that changes, the caller would misroute against an unscaled - // count, so fail loud in debug rather than clone silently. - Partitioning::Range(_) => { - debug_assert!( - false, - "scale_partitioning: range partitioning is not scaled at a boundary" - ); - partitioning.clone() - } + // A task-scaled range layout has no representation: the consumer side of a + // coalesce boundary sees every input task's ranges repeated, and + // `RangePartitioning` cannot express repeated split points. Cloning the layout + // unscaled would make the consumer request a partition set the producers never + // serve, so drop to an unknown layout with the scaled count; the boundary math + // and the consumer run on counts alone. + // TODO(#68): carry the range property across the boundary so consumer-side + // joins can stay co-partitioned. + Partitioning::Range(range) => Partitioning::UnknownPartitioning(f(range.partition_count())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::common::ScalarValue; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr, RangePartitioning, SplitPoint}; + + #[test] + fn range_partitioning_scales_to_an_unknown_layout() { + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("a", 0)), + Default::default(), + )]) + .unwrap(); + let splits = vec![ + SplitPoint::new(vec![ScalarValue::Int64(Some(10))]), + SplitPoint::new(vec![ScalarValue::Int64(Some(20))]), + ]; + let range = RangePartitioning::try_new(ordering, splits).unwrap(); + let scaled = scale_partitioning(&Partitioning::Range(range), |p| p * 4); + assert!(matches!(scaled, Partitioning::UnknownPartitioning(12))); } } From 9f38fbacf1797f17b09dd34349ece0df479ef6b1 Mon Sep 17 00:00:00 2001 From: Mithun Chicklore Yogendra Date: Sat, 8 Aug 2026 15:19:51 +0530 Subject: [PATCH 12/15] fix(shm): add task-aware multiplexing (#57) ## What Add task identity to shared-memory MPP data streams, allowing several tasks from the same stage to run on one producer process. ## Why DataFusion can plan more logical tasks than there are worker processes. When several tasks from one stage run on the same producer process and write to the same output partition, the prior transport identity `(sender_proc, stage_id, partition)` is ambiguous: one task EOF could close another task stream. Handling PostgreSQL launches with fewer attached workers additionally depends on [ParadeDB #5809](https://github.com/paradedb/paradedb/pull/5809), which creates the shared-memory mesh from the workers that actually attached. That gives the sender and receiver the same modulo base; this PR then keeps the logical task streams that share a producer process distinct. ## How - Extend data-frame identity to `(sender_proc, stage_id, task_id, partition)`. - Carry `task_id` through batch, EOF, cancellation, and oversized-frame chunk reassembly. - Register consumer streams with the same task-aware identity. - Permit `proc_for_task` to multiplex several tasks onto one worker safely. - Add a deterministic five-task/three-worker in-process query regression and a focused EOF-isolation test. ## Tests - `cargo +1.97.1 test -p datafusion-distributed --lib shm::transport::tests -- --nocapture` - `cargo +1.97.1 test -p datafusion-distributed --lib --no-run` - `cargo +1.97.1 check -p datafusion-distributed` --- src/shm/README.md | 10 +- src/shm/dsm.rs | 2 +- src/shm/in_process.rs | 171 +++++++++--- src/shm/mod.rs | 4 +- src/shm/runtime.rs | 67 +++-- src/shm/self_hosted.rs | 9 +- src/shm/setup.rs | 12 +- src/shm/sink.rs | 19 +- src/shm/transport.rs | 618 ++++++++++++++++++++++++++++------------- 9 files changed, 612 insertions(+), 300 deletions(-) diff --git a/src/shm/README.md b/src/shm/README.md index 6993fcc40..4dd64826b 100644 --- a/src/shm/README.md +++ b/src/shm/README.md @@ -6,7 +6,7 @@ This module provides a non-gRPC [`ChannelResolver`](../protocol/worker_channel.r ## Architecture Overview -The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC design while replacing network serialization and TCP sockets with Direct Shared Memory (DSM) ring buffers and compact 16-byte frame headers. +The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC design while replacing network serialization and TCP sockets with Direct Shared Memory (DSM) ring buffers and compact 20-byte frame headers. ```text ┌─────────────────────────────────────────────────────────────────────────────┐ @@ -20,7 +20,7 @@ The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC │ 1. Receives SetPlan & registers plan fragment (TaskKey) │ │ 2. Idles until ExecuteTaskFrame arrives from downstream consumer │ │ 3. ExecuteTaskFrame arrives specifying partition_range (start..end) │ -│ 4. Opens per-partition sinks lazily for start..end │ +│ 4. Opens per-task, per-partition sinks lazily for start..end │ │ 5. Evaluates run_worker_fragment(plan, sinks, ctx, start..end) │ │ 6. Streams batches out via DSM ring buffers │ └─────────────────────────────────────────────────────────────────────────────┘ @@ -31,17 +31,17 @@ The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC ## Control & Data Plane Design ### 1. Control Plane (`MppMesh`, `DrainHandle` & `run_execute_task_loop`) -- **Messages**: Control frames (`ExecuteTask`, `SetPlan`, `TaskMetrics`, `Cancel`, `EOF`) are tagged with a 16-byte [`MppFrameHeader`](./transport.rs) (`kind`, `stage_id`, `partition`, `sender_proc`) and routed through `MppMesh`. +- **Messages**: Frames carry a 20-byte [`MppFrameHeader`](./transport.rs): `kind`, `stage_id`, `task_id`, `partition`, and `sender_proc`. Control frames route a `(stage_id, task_number)` address; data frames (`Batch`, `EOF`, `Cancel`, and `Chunk`) route a `(stage_id, task_id, partition)` stream. - **Demand-Driven Pull Execution**: - Downstream consumers issue `ExecuteTaskFrame` to upstream producer tasks via `MppMesh::send_execute_task`. - Upstream worker tasks wait on `MppMesh::take_execute_task_rx(stage_num, task_idx)` for incoming execution requests before spawning task fragments. - Workers run the demand loop via [`run_execute_task_loop`](./setup.rs), which validates requested partition ranges (`start..end`), guards against duplicate/overlapping partition claims, listens to `CancellationToken` for prompt cancellation unwinding, and periodically drives inbound ring draining. -- **Demuxing**: Incoming frames are demuxed cooperatively by [`DrainHandle`](./transport.rs) into per-`(stage_id, task_number)` request registries and per-channel record batch buffers. +- **Demuxing**: Incoming frames are demuxed cooperatively by [`DrainHandle`](./transport.rs) into per-`(stage_id, task_number)` request registries and per-`(sender_proc, stage_id, task_id, partition)` record-batch buffers. ### 2. Data Plane (DSM Ring Buffers) - Output partitions write record batches into lock-free/cache-aligned DSM ring buffers (`mpsc_ring.rs`). - Batches are serialized as Arrow IPC streams directly into ring slots. -- Dedicated per-partition rings eliminate head-of-line blocking across tasks. +- One shared inbox per process multiplexes data streams; the task-aware stream key keeps tasks on the same producer process independent. --- diff --git a/src/shm/dsm.rs b/src/shm/dsm.rs index 6dca74aa5..f01e0d619 100644 --- a/src/shm/dsm.rs +++ b/src/shm/dsm.rs @@ -253,7 +253,7 @@ fn ring_dims_for(queue_bytes: usize) -> (u32, u32) { /// The own-inbox is the multiplexed entry point: every peer attaches to it as a sender /// (each `DsmMpscSender` increments the ring's `sender_count`) and stamps its identity /// into `MppFrameHeader::sender_proc` on every frame. The receiver side pulls frames -/// off that single ring and routes them to per-`(sender_proc, stage_id, partition)` +/// off that single ring and routes them to per-`(sender_proc, stage_id, task_id, partition)` /// channel buffers via [`DrainHandle`]. pub(super) struct ProcAttach { /// `outbound_senders[i]` writes to peer `peer_proc_for_index(this_proc, i)`'s inbox. diff --git a/src/shm/in_process.rs b/src/shm/in_process.rs index f3fd0d20b..e4a8ecdb9 100644 --- a/src/shm/in_process.rs +++ b/src/shm/in_process.rs @@ -66,7 +66,7 @@ use super::mpsc_ring::Wakeup; use super::runtime::{InProcessWorkerResolver, MppMesh, ShmChannelResolver, proc_for_task}; use super::setup::{collect_task_metrics, dsm_region_bytes, leader_setup, worker_setup}; use super::transport::{ - CooperativeDrainSet, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, + CooperativeDrainSet, MppDataStreamKey, MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, }; /// Per-inbox DSM ring size for the in-process mesh. Generous: the test ships a handful of tiny @@ -363,7 +363,7 @@ fn fragment_task_ctx( } /// Run all fragments owned by one worker proc, then signal completion. Mirrors the body of -/// pg_search's `run_mpp_worker`: build a [`WorkerSink`] that routes by partition, open a +/// pg_search's `run_mpp_worker`: build a [`WorkerSink`] that routes by task and partition, open a /// [`PartitionSink`] per output partition, execute each dispatched fragment as-is (it arrives /// ready-to-run, nested stages `Remote`, boundary leaves reading the mesh), and join. async fn run_worker_proc( @@ -387,7 +387,7 @@ async fn run_worker_proc( .map(|s| s.clone_with_header(MppFrameHeader::task_metrics(0, 0, mesh.this_proc))); // One sink serves every stage this proc produces; it owns the base outbound senders and routes - // each (stage, partition) to the destination proc's send end. + // each (stage, task, partition) to the destination proc's send end. let worker_sink = Arc::new(ShmMqWorkerSink { outbound, mesh: Arc::clone(&mesh), @@ -458,7 +458,11 @@ async fn run_worker_proc( async move { let mut task_sinks = Vec::with_capacity(range.len()); for q in range.clone() { - task_sinks.push(worker_sink.open_partition(stage_id as usize, q)?); + task_sinks.push(worker_sink.open_partition( + stage_id as usize, + task_idx as usize, + q, + )?); } crate::shm::setup::run_worker_fragment(plan, task_sinks, task_ctx, range) .await @@ -487,7 +491,7 @@ async fn run_worker_proc( Ok(()) } -/// Test-harness [`WorkerSink`]: routes each `(stage, partition)` to the destination proc's outbound +/// Test-harness [`WorkerSink`]: routes each `(stage, task, partition)` to the destination proc's outbound /// send end, the in-process analog of what pg_search builds on a real backend. Holds the base /// senders plus the per-stage routing so `open_partition` reproduces the header + cooperative-drain /// wiring the produce loop used to apply inline. @@ -499,7 +503,12 @@ struct ShmMqWorkerSink { } impl WorkerSink for ShmMqWorkerSink { - fn open_partition(&self, stage: usize, partition: usize) -> Result> { + fn open_partition( + &self, + stage: usize, + task: usize, + partition: usize, + ) -> Result> { let routing = self.routing.get(&(stage as u32)).ok_or_else(|| { DataFusionError::Internal(format!("run_worker_proc: no routing for stage {stage}")) })?; @@ -516,8 +525,15 @@ impl WorkerSink for ShmMqWorkerSink { })?; let sender = base .clone_with_header(MppFrameHeader::batch( - stage as u32, - partition as u32, + MppDataStreamKey::new( + stage as u32, + u32::try_from(task).map_err(|_| { + DataFusionError::Internal(format!( + "run_worker_proc: task {task} exceeds transport u32" + )) + })?, + partition as u32, + ), self.mesh.this_proc, )) .with_cooperative_drain(Arc::clone(&self.mesh) as Arc); @@ -594,8 +610,8 @@ mod tests { use datafusion::datasource::MemTable; use futures::TryStreamExt; - /// Total procs = leader (proc 0) + `N_WORKERS` producers. With round-robin `proc_for_task`, - /// worker proc `p` runs producer task `p - 1`. + /// Total procs = leader (proc 0) + `N_WORKERS` producers. `proc_for_task` maps task `i` to + /// proc `1 + i % N_WORKERS`, so one worker can own multiple logical tasks. const N_WORKERS: u32 = 3; fn table_schema() -> SchemaRef { @@ -605,10 +621,10 @@ mod tests { ])) } - /// `N_WORKERS` partitions, two rows each, so the shard estimator hands one partition per task. - fn table_partitions() -> Vec> { + /// Two rows per partition, so the shard estimator can hand one partition to each task. + fn table_partitions(n_partitions: i32) -> Vec> { let schema = table_schema(); - (0..N_WORKERS as i32) + (0..n_partitions) .map(|p| { let ids = Int32Array::from(vec![p * 2, p * 2 + 1]); let vals = Int32Array::from(vec![p * 20, p * 20 + 10]); @@ -620,8 +636,8 @@ mod tests { .collect() } - fn register_table(ctx: &SessionContext) { - let table = MemTable::try_new(table_schema(), table_partitions()).unwrap(); + fn register_table_with_partitions(ctx: &SessionContext, n_partitions: i32) { + let table = MemTable::try_new(table_schema(), table_partitions(n_partitions)).unwrap(); ctx.register_table("t", Arc::new(table)).unwrap(); } @@ -675,16 +691,19 @@ mod tests { let consumer = Arc::clone(&boot.workers[0].1); // proc 1 let producer = Arc::clone(&boot.workers[1].1); // proc 2 - assert!(!producer.stream_cancelled(7, 0)); + let cancelled = MppDataStreamKey::new(7, 0, 0); + let sibling_task = MppDataStreamKey::new(7, 3, 0); + assert!(!producer.stream_cancelled(cancelled)); - // Proc 1 abandons the `(stage 7, partition 0)` stream it reads from proc 2. - consumer.cancel_stream(2, 7, 0); + // Proc 1 abandons task 0's `(stage 7, partition 0)` stream it reads from proc 2. + consumer.cancel_stream(2, cancelled); // Proc 2 drains its inbox and sees the cancel its consumer sent. producer.try_drain_pass().unwrap(); - assert!(producer.stream_cancelled(7, 0)); - // Scoped to that one stream: a sibling partition stays live. - assert!(!producer.stream_cancelled(7, 1)); + assert!(producer.stream_cancelled(cancelled)); + // Scoped to that one stream: a sibling task using the same stage and output partition + // stays live. + assert!(!producer.stream_cancelled(sibling_task)); } /// `dispatch_capture` is Some on the leader session only: its coordinator is the one that @@ -694,12 +713,28 @@ mod tests { mesh: Arc, dispatch_capture: Option, ) -> SessionContext { - let config = SessionConfig::new().with_target_partitions(N_WORKERS as usize); + build_session_with_worker_and_partitions( + mesh, + dispatch_capture, + N_WORKERS as usize, + N_WORKERS as usize, + N_WORKERS as i32, + ) + } + + fn build_session_with_worker_and_partitions( + mesh: Arc, + dispatch_capture: Option, + planner_workers: usize, + target_partitions: usize, + table_partitions: i32, + ) -> SessionContext { + let config = SessionConfig::new().with_target_partitions(target_partitions); let mut builder = SessionStateBuilder::new() .with_default_features() .with_config(config) .with_distributed_option_extension(DistributedConfig::default()) - .with_distributed_worker_resolver(InProcessWorkerResolver::new(N_WORKERS as usize)) + .with_distributed_worker_resolver(InProcessWorkerResolver::new(planner_workers)) .with_distributed_channel_resolver(ShmChannelResolver::new(mesh)) .with_distributed_planner(); if let Some(captured) = dispatch_capture { @@ -719,7 +754,7 @@ mod tests { ); let state = builder.build(); let ctx = SessionContext::new_with_state(state); - register_table(&ctx); + register_table_with_partitions(&ctx, table_partitions); ctx } @@ -749,12 +784,13 @@ mod tests { /// PG worker runs, and it's exactly the cooperative model the transport is built for (the /// producer send spin and the consumer pull loop interleave by yielding, not by parallelism). #[tokio::test(flavor = "current_thread")] - async fn in_process_distributed_query_matches_serial() { + async fn in_process_distributed_query_multiplexes_tasks_without_losing_eofs() { let query = "SELECT id, val FROM t ORDER BY id"; + const N_TASKS: usize = 5; // Serial reference: same query, no distribution. let serial_ctx = SessionContext::new(); - register_table(&serial_ctx); + register_table_with_partitions(&serial_ctx, N_TASKS as i32); let expected = serial_ctx .sql(query) .await @@ -763,7 +799,7 @@ mod tests { .await .unwrap(); let expected_ids = ids_of(&expected); - assert_eq!(expected_ids, vec![0, 1, 2, 3, 4, 5]); + assert_eq!(expected_ids, (0..(N_TASKS as i32 * 2)).collect::>()); // One heap region stands in for the DSM segment; size it for n_procs = leader + workers. let n_procs = N_WORKERS + 1; @@ -810,7 +846,13 @@ mod tests { // Build the distributed plan once on the leader session; producers and consumer share it. let captured = new_captured_plans(); - let leader_ctx = build_session(Arc::clone(&leader_mesh), Some(Arc::clone(&captured))); + let leader_ctx = build_session_with_worker_and_partitions( + Arc::clone(&leader_mesh), + Some(Arc::clone(&captured)), + N_TASKS, + N_TASKS, + N_TASKS as i32, + ); let physical = leader_ctx .sql(query) .await @@ -825,14 +867,26 @@ mod tests { ); let entries = collect_dispatched_stages(&physical, N_WORKERS); - // Guard against a planner change that silently collapses the query to a trivial one-task - // gather: the producer stage must actually fan across every worker, or the transport's - // multi-task routing never gets exercised. + // There are five producer tasks but only three workers. Round-robin assignment puts + // tasks 0 and 3 on worker 1, and they both send stage output partition 0 to the leader. + // The task id in the transport header is what keeps task 0's EOF from closing task 3. assert!( - entries.iter().any(|e| e.task_count == N_WORKERS as usize), - "expected a producer stage with task_count = {N_WORKERS}; got {:?}", + entries.iter().any(|e| e.task_count == N_TASKS), + "expected a producer stage with task_count = {N_TASKS}; got {:?}", entries.iter().map(|e| e.task_count).collect::>() ); + let multiplexed = entries + .iter() + .find(|e| e.task_count == N_TASKS) + .expect("five-task producer stage"); + assert_eq!( + fragments_for_proc(std::slice::from_ref(multiplexed), 1, N_WORKERS) + .iter() + .map(|fragment| fragment.task_idx) + .collect::>(), + vec![0, 3], + "worker 1 must own two populated producer tasks" + ); // Launch the producer fragments before the leader pulls, so the mesh has data flowing while // the consumer drains. On the current-thread runtime the spawned tasks interleave with the @@ -840,7 +894,13 @@ mod tests { let mut workers = JoinSet::new(); for (proc_idx, mesh, outbound) in worker_setups { let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); - let session = build_session(Arc::clone(&mesh), None); + let session = build_session_with_worker_and_partitions( + Arc::clone(&mesh), + None, + N_WORKERS as usize, + N_TASKS, + N_WORKERS as i32, + ); workers.spawn(run_worker_proc( fragments, outbound, @@ -972,15 +1032,17 @@ mod tests { } } - /// A `GROUP BY` plans a nested `NetworkShuffleExec`, so this exercises hash-routed - /// worker-to-worker traffic and the self-loop sender, which the plain gather test never - /// touches. That routing is the main thing an upstream rebase can silently break. + /// A `GROUP BY` plans a nested `NetworkShuffleExec`, so this combines hash-routed + /// worker-to-worker traffic with five logical tasks on three worker processes. It exercises + /// the self-loop sender and the task-aware streams that plain gather and non-multiplexed + /// shuffle tests miss independently. #[tokio::test(flavor = "current_thread")] - async fn in_process_shuffle_query_matches_serial() { + async fn in_process_multiplexed_shuffle_query_matches_serial() { let query = "SELECT val, count(*) AS c FROM t GROUP BY val ORDER BY val"; + const N_TASKS: usize = 5; let serial_ctx = SessionContext::new(); - register_table(&serial_ctx); + register_table_with_partitions(&serial_ctx, N_TASKS as i32); let expected = serial_ctx .sql(query) .await @@ -991,7 +1053,13 @@ mod tests { let boot = bootstrap_mesh(N_WORKERS + 1); let captured = new_captured_plans(); - let leader_ctx = build_session(Arc::clone(&boot.leader_mesh), Some(Arc::clone(&captured))); + let leader_ctx = build_session_with_worker_and_partitions( + Arc::clone(&boot.leader_mesh), + Some(Arc::clone(&captured)), + N_TASKS, + N_TASKS, + N_TASKS as i32, + ); let physical = leader_ctx .sql(query) .await @@ -1007,11 +1075,32 @@ mod tests { "expected a hash-routed producer stage; got {:?}", entries.iter().map(|e| &e.routing).collect::>() ); + let multiplexed = entries + .iter() + .find(|entry| { + entry.task_count == N_TASKS + && matches!(entry.routing, FragmentRouting::Hashed { .. }) + }) + .expect("five-task hash-routed producer stage"); + assert_eq!( + fragments_for_proc(std::slice::from_ref(multiplexed), 1, N_WORKERS) + .iter() + .map(|fragment| fragment.task_idx) + .collect::>(), + vec![0, 3], + "worker 1 must own two shuffled producer tasks" + ); let mut workers = JoinSet::new(); for (proc_idx, mesh, outbound) in boot.workers { let fragments = fragments_for_proc(&entries, proc_idx, N_WORKERS); - let session = build_session(Arc::clone(&mesh), None); + let session = build_session_with_worker_and_partitions( + Arc::clone(&mesh), + None, + N_WORKERS as usize, + N_TASKS, + N_WORKERS as i32, + ); workers.spawn(run_worker_proc( fragments, outbound, diff --git a/src/shm/mod.rs b/src/shm/mod.rs index 0c2e2f37f..2ef41a293 100644 --- a/src/shm/mod.rs +++ b/src/shm/mod.rs @@ -71,8 +71,8 @@ pub use setup::{ }; pub use sink::{PartitionSink, WorkerSink}; pub use transport::{ - CooperativeDrainSet, ExecuteTaskFrame, ExecuteTaskRx, Interrupt, MppFrameHeader, - MppPartitionSink, MppSender, NoInterrupt, SendBatchStats, SetPlanFrame, + CooperativeDrainSet, ExecuteTaskFrame, ExecuteTaskRx, Interrupt, MppDataStreamKey, + MppFrameHeader, MppPartitionSink, MppSender, NoInterrupt, SendBatchStats, SetPlanFrame, }; /// Out-of-DSM liveness flag shared by the ring handles from one attach. The embedder flips it to diff --git a/src/shm/runtime.rs b/src/shm/runtime.rs index 55bf6c49b..3d85cb53f 100644 --- a/src/shm/runtime.rs +++ b/src/shm/runtime.rs @@ -54,8 +54,8 @@ use crate::{ use super::AliveFlag; use super::transport::{ - CooperativeDrainSet, DrainHandle, DrainItem, Interrupt, MppFrameHeader, MppSender, - SendBatchStats, SetPlanFrame, + CooperativeDrainSet, DrainHandle, DrainItem, Interrupt, MppDataStreamKey, MppFrameHeader, + MppSender, SendBatchStats, SetPlanFrame, }; /// A proc's outbound senders to each peer inbox, shared between the mesh (for `Cancel` frames) and @@ -67,10 +67,8 @@ pub type PeerSenders = Arc>>>; /// (consumer-only), workers are `1..n_procs` (each hosts producer fragments). /// /// A stage's task count is set by the DF-D task estimator chain, not by the worker proc count. -/// The transport does NOT support more tasks than producer procs: channels are keyed -/// `(sender_proc, stage, partition)`, so two tasks wrapped onto one proc would interleave on one -/// channel and the first EOF would truncate the other task's output. `ShmWorkerChannel::execute_task` -/// rejects that shape; the modulo here only keeps the function total. +/// Several tasks may share a worker; their data streams remain independent because the transport +/// key includes `task_id` in addition to the sender proc, stage, and output partition. #[inline] pub fn proc_for_task(n_workers: u32, task_idx: u32) -> u32 { 1 + (task_idx % n_workers.max(1)) @@ -81,7 +79,7 @@ pub fn proc_for_task(n_workers: u32, task_idx: u32) -> u32 { /// Each process owns one MPSC inbox in DSM that receives frames from every peer. /// `inbound_receiver` consolidates that inbox plus the in-proc self-loop channel (for /// producer-and-consumer-on-same-worker fragments) into a single `DrainHandle`. Frames -/// carry `(sender_proc, stage_id, partition)` in their header so the routing registry +/// carry `(sender_proc, stage_id, task_id, partition)` in their header so the routing registry /// inside the handle delivers each frame to the matching consumer. /// /// [`MppFrameHeader`]: super::transport::MppFrameHeader @@ -94,7 +92,7 @@ pub struct MppMesh { pub n_procs: u32, /// Single cooperative inbound handle pulling every frame addressed to this proc. The /// DSM MPSC inbox and an in-proc self-loop receiver both feed into this handle. Demux - /// to per-`(sender_proc, stage_id, partition)` channel buffers happens inside via + /// to per-`(sender_proc, stage_id, task_id, partition)` channel buffers happens inside via /// `DrainHandle::register_channel`. pub(super) inbound_receiver: Arc, /// Cancellation hook, injected by the embedder, checked at the transport's block points (the @@ -220,26 +218,26 @@ impl MppMesh { .take_execute_task_rx(stage_id, task_number) } - /// Tell the producer on `producer_proc` to stop the `(stage_id, partition)` stream: this proc's + /// Tell the producer on `producer_proc` to stop the `(stage_id, task_id, partition)` stream: this proc's /// consumer of it stopped reading before EOF. Ships one `Cancel` frame, leaving the rings /// healthy for metrics and every other stream. A no-op when no senders are installed (the /// embedder hasn't wired this proc, or teardown cleared them). /// - /// Stream-level so any consumer can cancel its own input: every `(producer_proc, stage, + /// Stream-level so any consumer can cancel its own input: every `(producer_proc, stage, task, /// partition)` channel has a single consumer, so one stream's drop never cuts off a sibling's. - pub fn cancel_stream(&self, producer_proc: u32, stage_id: u32, partition: u32) { + pub fn cancel_stream(&self, producer_proc: u32, stream: MppDataStreamKey) { let guard = self.cancel_senders.lock().unwrap(); let Some(senders) = guard.as_ref() else { return; }; let senders = senders.lock().unwrap(); if let Some(Some(sender)) = senders.get(producer_proc as usize) { - sender.try_send_cancel(stage_id, partition); + sender.try_send_cancel(stream); } } /// The single cooperative inbound handle that pulls frames from every peer (and the - /// self-loop) into per-`(sender_proc, stage_id, partition)` channel buffers. + /// self-loop) into per-`(sender_proc, stage_id, task_id, partition)` channel buffers. pub fn inbound_receiver(&self) -> &Arc { &self.inbound_receiver } @@ -318,8 +316,8 @@ impl CooperativeDrainSet for MppMesh { self.interrupt.check() } - fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { - self.inbound_receiver.stream_cancelled(stage_id, partition) + fn stream_cancelled(&self, stream: MppDataStreamKey) -> bool { + self.inbound_receiver.stream_cancelled(stream) } } @@ -351,7 +349,7 @@ impl ChannelResolver for ShmChannelResolver { /// A [`WorkerChannel`] over the mesh. `execute_task((stage, task), partition_range)` translates the /// DF-D `(stage, task)` addressing into the proc-pair grid: `proc_for_task(n_workers, task_number)` /// selects which `sender_proc` hosts the producer-side task, and each returned stream pulls that -/// proc's `(sender_proc, stage_id, partition)` slice from the inbound drain. +/// task's `(sender_proc, stage_id, task_id, partition)` slice from the inbound drain. struct ShmWorkerChannel { mesh: Arc, } @@ -434,10 +432,8 @@ impl WorkerChannel for ShmWorkerChannel { )) })?; let sender_proc = proc_for_task(self.mesh.n_workers(), task_number); - // More tasks than producer procs would fold two tasks onto one - // `(sender_proc, stage, partition)` channel: interleaved batches, and the first task's EOF - // truncates the second. With no input_stage here to count tasks, the equivalent guard is - // that the routed proc stays in range, as the old code also checked. + // Several tasks can fold onto this sender proc. `task_number` remains part of the stream + // identity, while this range check protects the mesh lookup itself. if sender_proc >= self.mesh.n_procs { return Err(DataFusionError::Internal(format!( "ShmWorkerChannel: sender_proc={sender_proc} >= n_procs={} \ @@ -508,8 +504,7 @@ impl WorkerChannel for ShmWorkerChannel { streams.push(pull_partition_stream( Arc::clone(&self.mesh), sender_proc, - stage_id, - partition_u32, + MppDataStreamKey::new(stage_id, task_number, partition_u32), )); } Ok(streams) @@ -525,7 +520,7 @@ impl WorkerChannel for ShmWorkerChannel { } } -/// Build the cooperative pull-loop stream for one `(producer_proc, stage_id, partition)` channel. +/// Build the cooperative pull-loop stream for one `(producer_proc, stage_id, task_id, partition)` channel. /// /// The inbound drain runs inline on the consumer's thread. Each iteration checks for cancellation /// (via the injected interrupt extension point), drains the receiver into the registry, pops one @@ -534,26 +529,28 @@ impl WorkerChannel for ShmWorkerChannel { fn pull_partition_stream( mesh: Arc, producer_proc: u32, - stage_id: u32, - partition: u32, + stream: MppDataStreamKey, ) -> BoxStream<'static, Result> { // One drain per process, shared across all sender_procs. The channel-buffer registry keys by - // (sender_proc, stage_id, partition) so this consumer still sees only its named sender's slice + // `(sender_proc, stage_id, task_id, partition)` so this consumer still sees only its named + // sender's slice // even though the underlying inbox is shared with all peers. let drain = Arc::clone(mesh.inbound_receiver()); log::debug!( - "shm transport execute: register channel sender_proc={producer_proc} stage_id={stage_id} \ - partition={partition}" + "shm transport execute: register channel sender_proc={producer_proc} \ + stage_id={} task_id={} partition={}", + stream.stage_id, + stream.task_id, + stream.partition, ); - let buffer = drain.register_channel(producer_proc, stage_id, partition); + let buffer = drain.register_data_channel(producer_proc, stream); let stream = async_stream::stream! { // Any consumer cancels its own input stream when it drops early. The mesh no-ops the send // until the embedder wires this proc's outbound senders. let mut cancel_guard = StreamCancelGuard { mesh: Arc::clone(&mesh), producer_proc, - stage_id, - partition, + stream, armed: true, }; loop { @@ -613,7 +610,7 @@ impl WorkerResolver for InProcessWorkerResolver { } } -/// Cancels one `(stage_id, partition)` stream if its consumer drops before EOF, telling the +/// Cancels one `(stage_id, task_id, partition)` stream if its consumer drops before EOF, telling the /// producer on `producer_proc` to stop. A consumer that stops pulling early (a top-N `LIMIT`, an /// inner merge join exhausting a side, etc.) would otherwise leave that producer spinning on the /// full inbox until the statement timeout. Disarmed on a clean EOF: there the producer already @@ -621,16 +618,14 @@ impl WorkerResolver for InProcessWorkerResolver { struct StreamCancelGuard { mesh: Arc, producer_proc: u32, - stage_id: u32, - partition: u32, + stream: MppDataStreamKey, armed: bool, } impl Drop for StreamCancelGuard { fn drop(&mut self) { if self.armed { - self.mesh - .cancel_stream(self.producer_proc, self.stage_id, self.partition); + self.mesh.cancel_stream(self.producer_proc, self.stream); } } } diff --git a/src/shm/self_hosted.rs b/src/shm/self_hosted.rs index db0f473fb..b55ffce8e 100644 --- a/src/shm/self_hosted.rs +++ b/src/shm/self_hosted.rs @@ -59,8 +59,8 @@ use super::mpsc_ring::Wakeup; use super::runtime::{MppMesh, ShmMqWorkerTransport, proc_for_task}; use super::setup::{dsm_region_bytes, leader_setup, worker_setup}; use super::transport::{ - CooperativeDrainSet, Interrupt, MppFrameHeader, MppPartitionSink, MppSender, SendBatchStats, - SetPlanFrame, + CooperativeDrainSet, Interrupt, MppDataStreamKey, MppFrameHeader, MppPartitionSink, MppSender, + SendBatchStats, SetPlanFrame, }; use crate::proto as pb; use crate::{ @@ -767,7 +767,10 @@ impl QueryHarness { ); }; let sender = base - .clone_with_header(MppFrameHeader::batch(stage_num, q as u32, proc)) + .clone_with_header(MppFrameHeader::batch( + MppDataStreamKey::new(stage_num, task_i as u32, q as u32), + proc, + )) .with_cooperative_drain(Arc::clone(mesh) as Arc); sinks.push(Box::new(MppPartitionSink::new(sender))); } diff --git a/src/shm/setup.rs b/src/shm/setup.rs index 21a59a029..47832c7f0 100644 --- a/src/shm/setup.rs +++ b/src/shm/setup.rs @@ -45,8 +45,8 @@ use super::mesh::{DsmInboxReceiver, DsmInboxSender}; use super::mpsc_ring::{DsmMpscSender, NO_RECEIVER_TOKEN, Wakeup}; use super::runtime::MppMesh; use super::transport::{ - BatchChannelSender, DrainHandle, ExecuteTaskRx, Interrupt, MppFrameHeader, MppReceiver, - MppSender, ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, + BatchChannelSender, DrainHandle, ExecuteTaskRx, Interrupt, MppDataStreamKey, MppFrameHeader, + MppReceiver, MppSender, ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, }; use crate::proto as pb; use crate::work_unit_feed::RemoteWorkUnitFeedRegistry; @@ -107,14 +107,14 @@ fn build_outbound_senders( Arc::new(DsmInboxSender::new(dsm_send.to_control())); cancel[target_proc as usize] = Some(MppSender::with_header( control, - MppFrameHeader::batch(0, 0, this_proc), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), this_proc), )); let shared: Arc = Arc::new(DsmInboxSender::new(dsm_send)); senders[target_proc as usize] = Some(MppSender::with_header( shared, // Stamp `sender_proc = this_proc` so a stray frame that escapes the dispatcher's // `clone_with_header` overwrite still identifies its origin on the drain side. - MppFrameHeader::batch(0, 0, this_proc), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), this_proc), )); } (senders, cancel) @@ -304,11 +304,11 @@ pub unsafe fn worker_setup( let self_tx_arc: Arc = Arc::new(self_tx); outbound[proc_idx as usize] = Some(MppSender::with_header( Arc::clone(&self_tx_arc), - MppFrameHeader::batch(0, 0, proc_idx), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), proc_idx), )); cancel[proc_idx as usize] = Some(MppSender::with_header( Arc::clone(&self_tx_arc), - MppFrameHeader::batch(0, 0, proc_idx), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), proc_idx), )); let inbox = DsmInboxReceiver::new(attach.inbound_receiver); diff --git a/src/shm/sink.rs b/src/shm/sink.rs index a6ae19d7e..90dae276b 100644 --- a/src/shm/sink.rs +++ b/src/shm/sink.rs @@ -38,12 +38,17 @@ pub trait PartitionSink: Send { /// output batch in. The shared-memory mesh provides the implementation; it is constructed by the /// producer (which knows the per-partition routing), not handed out by the consume-side transport. pub trait WorkerSink: Send + Sync { - /// Takes `stage` and `partition` separately because one sink serves every stage, unlike the - /// per-stage read connection that closes over its stage. + /// Takes `stage`, producing `task`, and `partition` separately because one sink serves every + /// stage, unlike the per-stage read connection that closes over its stage. /// - /// `stage` is the producing stage's number and `partition` the producer task's own output - /// partition index, before routing. Several producer tasks of one stage may hold sinks for the - /// same pair, and the consumer merges them, so one `finish` is one producer task's EOF, not - /// channel completion (which stays transport-defined). - fn open_partition(&self, stage: usize, partition: usize) -> Result>; + /// `stage` is the producing stage's number, `task` its logical task number, and `partition` + /// the producer task's own output partition index before routing. Several producer tasks of + /// one stage may hold sinks for the same partition, and each needs a distinct stream identity: + /// one task's `finish` must never close another task's stream. + fn open_partition( + &self, + stage: usize, + task: usize, + partition: usize, + ) -> Result>; } diff --git a/src/shm/transport.rs b/src/shm/transport.rs index 2604f6ad3..23e7c9215 100644 --- a/src/shm/transport.rs +++ b/src/shm/transport.rs @@ -17,8 +17,8 @@ //! Transport layer for MPP shuffle. //! -//! - [`MppFrameHeader`]: fixed 16-byte prefix tagging each wire message with -//! `(stage_id, partition)`, so one queue carries frames for many logical channels. +//! - [`MppFrameHeader`]: fixed 20-byte prefix tagging each wire message with +//! `(stage_id, task_id, partition)`, so one queue carries frames for many logical channels. //! - [`encode_frame_into`] / [`decode_frame`]: Arrow IPC serialize/deserialize with //! header prefix. Only codec entry points; tests round-trip through the same path. //! - [`DrainBuffer`]: per-proc queue the drain writes into and the DataFusion consumer @@ -54,13 +54,13 @@ const MPP_FRAME_MAGIC: u32 = 0x4D505046; /// Wire-format size of [`MppFrameHeader`] in bytes. Asserted at compile time /// below via `const _: ()`. -const MPP_FRAME_HEADER_SIZE: usize = 16; +const MPP_FRAME_HEADER_SIZE: usize = 20; /// Kind of payload following [`MppFrameHeader`]. /// /// `Batch` is the common case. The header is followed by an Arrow IPC stream containing one /// `RecordBatch`. `Eof` carries no payload. It signals the receiver that the named -/// `(stage_id, partition)` channel is done, even though the underlying shm_mq queue may still +/// `(stage_id, task_id, partition)` channel is done, even though the underlying shm_mq queue may still /// carry frames for other channels. /// /// The remaining kinds are the control plane riding the same rings. For them the header's @@ -80,7 +80,7 @@ pub(super) enum MppFrameKind { FeedEof = 3, TaskMetrics = 4, SetPlan = 5, - /// Consumer -> producer: stop producing the `(stage_id, partition)` stream, the consumer + /// Consumer -> producer: stop producing the `(stage_id, task_id, partition)` stream, the consumer /// stopped reading it before EOF (a top-N `LIMIT`, an inner merge join exhausting a side, etc.). /// The producer ends that one stream cleanly. Scoped to the stream, not the connection, so the /// ring stays healthy for metrics and every other stream, the way gRPC closes one stream @@ -88,7 +88,7 @@ pub(super) enum MppFrameKind { Cancel = 6, /// One piece of a frame too large for the channel's frame bound. The payload is a /// `[total_len: u64][offset: u64]` prefix followed by that byte range of the inner frame. - /// The receiver reassembles per `(sender_proc, stage_id, partition)` and decodes the inner + /// The receiver reassembles per `(sender_proc, stage_id, task_id, partition)` and decodes the inner /// frame once `total_len` bytes have arrived, so no single ring-resident message ever /// exceeds the ring and frame size is never an error. Chunk = 7, @@ -216,9 +216,47 @@ impl ExecuteTaskFrame { } } -/// 16-byte prefix on every transport frame. +/// Identity of one task's output partition within a distributed stage. /// -/// The fixed layout `[magic, flags, stage_id, partition]` (4×u32) is what +/// Batches, EOF, cancellation, and data chunks must use this key. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MppDataStreamKey { + pub stage_id: u32, + pub task_id: u32, + pub partition: u32, +} + +impl MppDataStreamKey { + pub const fn new(stage_id: u32, task_id: u32, partition: u32) -> Self { + Self { + stage_id, + task_id, + partition, + } + } +} + +/// One data stream as it appears on a receiver's shared inbox. +/// +/// `sender_proc` distinguishes producers sharing an inbox, including the self-loop. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct PhysicalStreamKey { + sender_proc: u32, + stream: MppDataStreamKey, +} + +impl PhysicalStreamKey { + const fn new(sender_proc: u32, stream: MppDataStreamKey) -> Self { + Self { + sender_proc, + stream, + } + } +} + +/// 20-byte prefix on every transport frame. +/// +/// The fixed layout `[magic, flags, stage_id, task_id, partition]` (5×u32) is what /// senders prepend before the Arrow IPC stream bytes and what receivers /// parse before deciding which channel buffer the payload belongs to. /// @@ -226,12 +264,15 @@ impl ExecuteTaskFrame { #[repr(C)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct MppFrameHeader { - // Private so headers only come out of `batch()`/`eof()`: hand-built ones could bypass + // Private so headers only come out of constructors: hand-built ones could bypass // `pack_flags`'s sender bound and the reserved-bits invariant, and the consumer would // reject them at decode, far from the producer. pub(super) magic: u32, pub(super) flags: u32, pub(super) stage_id: u32, + /// Logical task that produced a data stream. Task-control frames leave this zero because + /// their existing `partition` slot encodes the target task number. + pub(super) task_id: u32, pub(super) partition: u32, } @@ -270,24 +311,26 @@ fn pack_flags(kind: MppFrameKind, sender_proc: u32) -> u32 { } impl MppFrameHeader { - /// Build a `Batch` header for the given `(stage_id, partition)` stamped with `sender_proc`. - pub fn batch(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + /// Build a `Batch` header for one data stream stamped with `sender_proc`. + pub fn batch(stream: MppDataStreamKey, sender_proc: u32) -> Self { Self { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::Batch, sender_proc), - stage_id, - partition, + stage_id: stream.stage_id, + task_id: stream.task_id, + partition: stream.partition, } } - /// Build an `Eof` header for the given `(stage_id, partition)` stamped with `sender_proc`. + /// Build an `Eof` header for the given data stream stamped with `sender_proc`. /// Carries no payload; receivers route it to the channel buffer's source-done counter. - pub fn eof(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + pub fn eof(stream: MppDataStreamKey, sender_proc: u32) -> Self { Self { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::Eof, sender_proc), - stage_id, - partition, + stage_id: stream.stage_id, + task_id: stream.task_id, + partition: stream.partition, } } @@ -298,6 +341,7 @@ impl MppFrameHeader { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::WorkUnit, sender_proc), stage_id, + task_id: 0, partition: task_number, } } @@ -308,6 +352,7 @@ impl MppFrameHeader { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::FeedEof, sender_proc), stage_id, + task_id: 0, partition: task_number, } } @@ -318,6 +363,7 @@ impl MppFrameHeader { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::TaskMetrics, sender_proc), stage_id, + task_id: 0, partition: task_number, } } @@ -329,6 +375,7 @@ impl MppFrameHeader { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::SetPlan, sender_proc), stage_id, + task_id: 0, partition: task_number, } } @@ -339,36 +386,45 @@ impl MppFrameHeader { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::ExecuteTask, sender_proc), stage_id, + task_id: 0, partition: task_number, } } - /// Build a `Cancel` header for the `(stage_id, partition)` stream, stamped with the consumer's - /// `sender_proc`. Carries no payload; the producer reads it as "stop sending this stream." - pub fn cancel(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + /// Build a `Cancel` header for a data stream, stamped with the consumer's `sender_proc`. + /// Carries no payload; the producer reads it as "stop sending this stream." + pub fn cancel(stream: MppDataStreamKey, sender_proc: u32) -> Self { Self { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::Cancel, sender_proc), - stage_id, - partition, + stage_id: stream.stage_id, + task_id: stream.task_id, + partition: stream.partition, } } - /// Build a `Chunk` header for one piece of an oversized frame. `stage_id`/`partition` - /// mirror the inner frame's addressing: together with `sender_proc` they key the + /// Build a `Chunk` header for one piece of an oversized frame. The data stream mirrors the + /// inner frame's addressing: together with `sender_proc` they key the /// receiver's reassembly, so chunked frames from two streams of the same proc can /// interleave without corrupting each other. - pub fn chunk(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + pub fn chunk(stream: MppDataStreamKey, sender_proc: u32) -> Self { Self { magic: MPP_FRAME_MAGIC, flags: pack_flags(MppFrameKind::Chunk, sender_proc), - stage_id, - partition, + stage_id: stream.stage_id, + task_id: stream.task_id, + partition: stream.partition, } } + /// Returns the data-stream address carried by a `Batch`, `Eof`, `Cancel`, or `Chunk` frame. + /// Control frames are a distinct tagged layout and keep their task address in `partition`. + pub(super) const fn data_stream(&self) -> MppDataStreamKey { + MppDataStreamKey::new(self.stage_id, self.task_id, self.partition) + } + /// The mesh peer that wrote this frame. The drain demuxes incoming frames into the - /// per-channel buffer registry by `(sender_proc, stage_id, partition)`. + /// per-channel buffer registry by `(sender_proc, stage_id, task_id, partition)`. pub fn sender_proc(&self) -> u32 { (self.flags >> FRAME_SENDER_SHIFT) & 0xFFFF } @@ -407,7 +463,8 @@ impl MppFrameHeader { out[0..4].copy_from_slice(&self.magic.to_le_bytes()); out[4..8].copy_from_slice(&self.flags.to_le_bytes()); out[8..12].copy_from_slice(&self.stage_id.to_le_bytes()); - out[12..16].copy_from_slice(&self.partition.to_le_bytes()); + out[12..16].copy_from_slice(&self.task_id.to_le_bytes()); + out[16..20].copy_from_slice(&self.partition.to_le_bytes()); } /// Parse from the first `MPP_FRAME_HEADER_SIZE` bytes of `bytes`. Returns @@ -438,17 +495,18 @@ impl MppFrameHeader { magic, flags: u32::from_le_bytes(bytes[4..8].try_into().unwrap()), stage_id: u32::from_le_bytes(bytes[8..12].try_into().unwrap()), - partition: u32::from_le_bytes(bytes[12..16].try_into().unwrap()), + task_id: u32::from_le_bytes(bytes[12..16].try_into().unwrap()), + partition: u32::from_le_bytes(bytes[16..20].try_into().unwrap()), }) } } -/// Serialize `batch` into `buf` with a 16-byte [`MppFrameHeader`] prefix -/// addressing it to `(stage_id, partition)`. Wire format: +/// Serialize `batch` into `buf` with a 20-byte [`MppFrameHeader`] prefix +/// addressing it to `(stage_id, task_id, partition)`. Wire format: /// /// ```text -/// [ magic | flags | stage_id | partition ] [ Arrow IPC stream bytes ] -/// |---------- 16 bytes --------| |---- variable ----| +/// [ magic | flags | stage_id | task_id | partition ] [ Arrow IPC stream bytes ] +/// |--------------- 20 bytes ---------------| |---- variable ----| /// ``` /// /// `flags` encodes kind + sender_proc; see the bit-layout block near @@ -517,23 +575,21 @@ fn compact_rows( Ok(RecordBatch::try_new(batch.schema(), columns)?) } -/// Serialize a payload-less [`MppFrameKind::Eof`] frame for `(stage_id, partition)` -/// into `buf`. The shm_mq peer reads this as a 16-byte message and routes it to +/// Serialize a payload-less [`MppFrameKind::Eof`] frame for `(stage_id, task_id, partition)` +/// into `buf`. The shm_mq peer reads this as a 20-byte message and routes it to /// the channel buffer's source-done counter without touching Arrow IPC. /// Consumed by [`MppSender::send_eof_traced`] when a producer fragment's -/// per-partition stream exhausts, so the receiver's `(stage_id, partition)` +/// per-partition stream exhausts, so the receiver's `(stage_id, task_id, partition)` /// channel buffer transitions to `Eof` even though the multiplexed shm_mq queue /// stays attached for other channels. fn encode_eof_frame_into( - stage_id: u32, - partition: u32, + stream: MppDataStreamKey, sender_proc: u32, buf: &mut Vec, ) -> Result<(), DataFusionError> { buf.clear(); buf.resize(MPP_FRAME_HEADER_SIZE, 0); - MppFrameHeader::eof(stage_id, partition, sender_proc) - .write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); + MppFrameHeader::eof(stream, sender_proc).write_to(&mut buf[..MPP_FRAME_HEADER_SIZE]); Ok(()) } @@ -558,7 +614,7 @@ fn encode_chunk_frame_into( } /// Parse a `Chunk` frame's payload into `(total_len, offset, data)`. The caller has already -/// parsed and validated the 16-byte frame header. +/// parsed and validated the 20-byte frame header. fn parse_chunk_frame(bytes: &[u8]) -> Result<(usize, usize, &[u8]), DataFusionError> { let payload = &bytes[MPP_FRAME_HEADER_SIZE..]; if payload.len() < CHUNK_PREFIX_SIZE { @@ -604,7 +660,7 @@ enum FrameBody { Cancel, } -/// Inverse of the frame encoders. Parses the 16-byte header and decodes the payload according +/// Inverse of the frame encoders. Parses the 20-byte header and decodes the payload according /// to the kind. Receivers branch on the body to decide routing. fn decode_frame(bytes: &[u8]) -> Result<(MppFrameHeader, FrameBody), DataFusionError> { let header = MppFrameHeader::parse(bytes)?; @@ -662,7 +718,7 @@ fn decode_frame(bytes: &[u8]) -> Result<(MppFrameHeader, FrameBody), DataFusionE /// variant) and the consumer that pops batches. /// /// In the cooperative path each `DrainBuffer` corresponds to one logical channel: one -/// `(stage_id, partition)` entry in the owning [`DrainHandle`]'s registry. `num_sources` is +/// `(stage_id, task_id, partition)` entry in the owning [`DrainHandle`]'s registry. `num_sources` is /// always `1` there because a given drain serves a single sender_proc, which is the only producer /// for any channel routed through it. The test-only thread path uses a single shared buffer with /// `num_sources = N` over an N-sender setup. @@ -851,10 +907,10 @@ pub trait CooperativeDrainSet: Send + Sync { Ok(()) } - /// Whether a consumer cancelled the `(stage_id, partition)` stream (a `Cancel` frame arrived on + /// Whether a consumer cancelled the `(stage_id, task_id, partition)` stream (a `Cancel` frame arrived on /// this proc's inbox). The send spin ends the producer's stream cleanly when it's set. Default /// `false` for drains that don't carry inbound control frames (in-proc test channels). - fn stream_cancelled(&self, _stage_id: u32, _partition: u32) -> bool { + fn stream_cancelled(&self, _stream: MppDataStreamKey) -> bool { false } } @@ -879,8 +935,8 @@ impl CooperativeDrainSet for DrainHandle { DrainHandle::try_drain_pass(self) } - fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { - DrainHandle::stream_cancelled(self, stage_id, partition) + fn stream_cancelled(&self, stream: MppDataStreamKey) -> bool { + DrainHandle::stream_cancelled(self, stream) } } @@ -894,13 +950,13 @@ impl CooperativeDrainSet for DrainHandle { /// un-stall. pub struct MppSender { /// Underlying byte channel. Held behind `Arc` so multiple `MppSender`s can share one - /// `shm_mq` queue while tagging frames with different `(stage_id, partition)` headers, which + /// `shm_mq` queue while tagging frames with different `(stage_id, task_id, partition)` headers, which /// is the multiplexed path's natural pattern. Clone the Arc, build a new `MppSender` with a /// different header, both write into the same queue. pub(super) channel: Arc, cooperative_drain: Option>, /// Frame header prepended to every outgoing batch. Identifies the logical - /// `(stage_id, partition)` channel the receiver demultiplexes on. Per-sender rather than + /// `(stage_id, task_id, partition)` channel the receiver demultiplexes on. Per-sender rather than /// per-call: each partition gets its own `MppSender` via `clone_with_header`, all sharing /// the underlying `Arc` of a single shm_mq queue. pub(super) header: MppFrameHeader, @@ -921,10 +977,6 @@ pub struct MppSender { unsafe impl Sync for MppSender {} impl MppSender { - /// Construct a sender that tags every outgoing batch with `header`. Production call sites - /// clone one shared `Arc` across N senders, each with a different - /// `MppFrameHeader::batch(stage, p)`. That's the multiplexed pattern for fanning multiple - /// partitions over one shm_mq queue. pub(super) fn with_header( channel: Arc, header: MppFrameHeader, @@ -950,13 +1002,18 @@ impl MppSender { } } - /// Whether the consumer of this sender's `(stage, partition)` stream cancelled it. Read by the + /// Whether the consumer of this sender's `(stage, task, partition)` stream cancelled it. Read by the /// produce loop to stop pulling its input, not just skip the send. `false` without a drain /// (in-proc test channels carry no inbound cancel). pub(super) fn stream_cancelled(&self) -> bool { + // A data-stream cancel can numerically match a control address; control traffic is never + // cancelled by a stream-level cancel. + if !matches!(self.header.kind(), Ok(MppFrameKind::Batch)) { + return false; + } self.cooperative_drain .as_ref() - .is_some_and(|d| d.stream_cancelled(self.header.stage_id, self.header.partition)) + .is_some_and(|d| d.stream_cancelled(self.header.data_stream())) } /// Attach a [`CooperativeDrainSet`] so `Self::send_batch_traced`'s spin @@ -996,10 +1053,10 @@ impl MppSender { result } - /// Send a payload-less [`MppFrameKind::Eof`] frame so the receiver's `(stage_id, partition)` + /// Send a payload-less [`MppFrameKind::Eof`] frame so the receiver's `(stage_id, task_id, partition)` /// channel buffer transitions to `Eof` and the consumer's pull loop terminates cleanly. /// - /// Producer fragments must call this exactly once per `(stage_id, partition)` channel after + /// Producer fragments must call this exactly once per `(stage_id, task_id, partition)` channel after /// the local stream exhausts. Without it the multiplexed shm_mq queue stays attached (other /// channels still flow) and the consumer channel buffer never reaches `sources_done == 1`. The /// receive-side [`DrainHandle::try_drain_pass`] decodes the frame and calls @@ -1025,7 +1082,7 @@ impl MppSender { result } - /// Bounded synchronous send of a `Cancel` frame for the `(stage_id, partition)` stream. The + /// Bounded synchronous send of a `Cancel` frame for the `(stage_id, task_id, partition)` stream. The /// consumer calls it on the sender to the producing proc when it abandons that stream. /// Synchronous so it can run from the consumer stream's drop, where `await` isn't available. /// @@ -1033,8 +1090,8 @@ impl MppSender { /// spin, so a slot frees and the frame lands well inside the bound even when the inbox is /// backed up. The bound only runs out if the producer already exited or died, and a dead worker /// makes the leader's wait-for-workers error out rather than hang. - pub fn try_send_cancel(&self, stage_id: u32, partition: u32) { - let header = MppFrameHeader::cancel(stage_id, partition, self.header.sender_proc()); + pub fn try_send_cancel(&self, stream: MppDataStreamKey) { + let header = MppFrameHeader::cancel(stream, self.header.sender_proc()); let mut buf = [0u8; MPP_FRAME_HEADER_SIZE]; header.write_to(&mut buf); for _ in 0..MAX_CONTROL_SEND_SPINS { @@ -1052,8 +1109,7 @@ impl MppSender { stats: &mut SendBatchStats, ) -> Result<(), DataFusionError> { encode_eof_frame_into( - self.header.stage_id, - self.header.partition, + self.header.data_stream(), self.header.sender_proc(), scratch, )?; @@ -1136,11 +1192,7 @@ impl MppSender { "mpp: channel frame bound {cap} cannot fit a chunk header" ))); }; - let header = MppFrameHeader::chunk( - self.header.stage_id, - self.header.partition, - self.header.sender_proc(), - ); + let header = MppFrameHeader::chunk(self.header.data_stream(), self.header.sender_proc()); let total_len = frame.len() as u64; let mut buf = Vec::with_capacity(MPP_FRAME_HEADER_SIZE + CHUNK_PREFIX_SIZE + chunk_data_max); @@ -1201,7 +1253,7 @@ impl MppSender { // The consumer cancelled this stream, so end the send cleanly and let the producer // fragment complete. Checked before the send so a cancel that landed mid-spin stops // the next iteration. - if drain.stream_cancelled(self.header.stage_id, self.header.partition) { + if self.stream_cancelled() { return Ok(false); } if self.spin_try_send_bytes(scratch).await? { @@ -1396,12 +1448,12 @@ impl crate::PartitionSink for MppPartitionSink { pub(super) struct MppReceiver { channel: Box, /// In-progress oversized frames, keyed by the chunk headers' `(sender_proc, stage_id, - /// partition)`. One entry per stream: a sender's frames for one stream are sequential, + /// task_id, partition)`. One entry per stream: a sender's frames for one stream are sequential, /// so its chunks never interleave with each other, while chunked frames of two streams /// from the same proc may. A prefix abandoned mid-frame (the producer observed the /// stream's cancel) sits here until the stream's `Eof` clears it or a fresh frame for /// the stream restarts at offset 0. - assemblies: Mutex>, + assemblies: Mutex>, } /// One stream's partially reassembled oversized frame. @@ -1465,11 +1517,13 @@ impl MppReceiver { if kind != MppFrameKind::Chunk { if kind == MppFrameKind::Eof { // The stream is over; drop any prefix its producer abandoned mid-frame. - self.assemblies.lock().unwrap().remove(&( - header.sender_proc(), - header.stage_id, - header.partition, - )); + self.assemblies + .lock() + .unwrap() + .remove(&PhysicalStreamKey::new( + header.sender_proc(), + header.data_stream(), + )); } return Ok(Some(bytes)); } @@ -1484,7 +1538,7 @@ impl MppReceiver { data.len() ))); } - let key = (header.sender_proc(), header.stage_id, header.partition); + let key = PhysicalStreamKey::new(header.sender_proc(), header.data_stream()); let mut assemblies = self.assemblies.lock().unwrap(); if offset == 0 { // Start of a new frame; replaces any abandoned predecessor for the same stream. @@ -1505,8 +1559,8 @@ impl MppReceiver { let Some(assembly) = assemblies.get_mut(&key) else { return Err(DataFusionError::Internal(format!( "mpp: continuation chunk (offset {offset}) with no first chunk for \ - sender {} stage {} partition {}", - key.0, key.1, key.2 + sender {} stage {} task {} partition {}", + key.sender_proc, key.stream.stage_id, key.stream.task_id, key.stream.partition, ))); }; if assembly.total_len != total_len || assembly.buf.len() != offset { @@ -1527,14 +1581,14 @@ impl MppReceiver { /// Decoded result of an [`MppReceiver::try_recv_batch`]. Carries the /// parsed [`MppFrameHeader`] so the drain thread can route the payload to -/// the right `(stage_id, partition)` channel buffer. +/// the right `(stage_id, task_id, partition)` channel buffer. #[derive(Debug)] pub(super) enum RecvBatchOutcome { Batch { header: MppFrameHeader, batch: RecordBatch, }, - /// A payload-less `Eof` frame for `header.(stage_id, partition)`. The + /// A payload-less `Eof` frame for `header.(stage_id, task_id, partition)`. The /// underlying shm_mq queue is still attached. The sender is just /// signalling that this logical channel is done, so we can EOF /// per-channel without dropping the whole queue. @@ -1565,7 +1619,7 @@ pub(super) enum RecvBatchOutcome { header: MppFrameHeader, metrics: pb::TaskMetrics, }, - /// Consumer abandoned the `header.(stage_id, partition)` stream; its producer stops sending it. + /// Consumer abandoned the `header.(stage_id, task_id, partition)` stream; its producer stops sending it. Cancel { header: MppFrameHeader, }, @@ -1574,7 +1628,7 @@ pub(super) enum RecvBatchOutcome { Error(DataFusionError), } -/// Per-`(sender_proc, stage_id, partition)` channel buffer registry owned by a cooperative +/// Per-`(sender_proc, stage_id, task_id, partition)` channel buffer registry owned by a cooperative /// [`DrainHandle`]. The handle may host several cooperative receivers (DSM MPSC inbox + self-loop /// in-proc), each demultiplexed by the [`MppFrameHeader`] prefix into the same `map`. /// `try_drain_pass` looks up the right channel buffer on every frame and pushes the payload into @@ -1585,11 +1639,11 @@ pub(super) enum RecvBatchOutcome { /// teardown unblock flows via [`DrainHandle::cancel_channel_buffers`] from the handle's `Drop`. #[derive(Default)] struct ChannelBufferRegistry { - /// Keyed by `(sender_proc, stage_id, partition)`. The unified inbox carries frames - /// from every peer, so each `(stage, partition)` consumer gets its own per-sender + /// Keyed by `(sender_proc, stage_id, task_id, partition)`. The unified inbox carries frames + /// from every peer, so each `(stage, task, partition)` consumer gets its own per-sender /// buffer. This preserves the implicit "one stream per sender" semantics that /// `WorkerConnection::execute` consumers rely on. - map: HashMap<(u32, u32, u32), Arc>, + map: HashMap>, /// Scopes whose receiver detached (or errored) before draining cleanly. Channels fed by a /// dead scope fail at registration time too, so a consumer that registers after the detach /// does not wait on a channel nothing will ever fill. @@ -1607,7 +1661,7 @@ pub(super) enum ReceiverScope { } /// Per-sender-proc drain: stashes the receivers and polls them inline from the cooperative spin -/// (no background thread), demuxing each frame into a per-`(stage_id, partition)` channel buffer. +/// (no background thread), demuxing each frame into a per-`(stage_id, task_id, partition)` channel buffer. /// /// Inline polling is the production requirement: pgrx's `check_active_thread` guard panics on any /// pg FFI call (including `shm_mq_receive`) from a non-backend thread, so the drain work has to @@ -1617,7 +1671,7 @@ pub(super) enum ReceiverScope { /// On drop, the handle cancels every channel buffer so any consumer blocked on `try_pop` unblocks /// with `Eof` — the drain can therefore never outlive its query, even on a panicked teardown. pub struct DrainHandle { - /// Per-(stage_id, partition) channel buffer registry. Populated lazily on first frame for a + /// Per-(stage_id, task_id, partition) channel buffer registry. Populated lazily on first frame for a /// channel, or up-front by callers (e.g. `WorkerConnection::execute`) that need a /// buffer to wait on before any frame arrives. channel_buffers: Mutex, @@ -1649,11 +1703,11 @@ pub struct DrainHandle { set_plan_registry: Mutex, /// Worker-side destination of `ExecuteTask` frames, keyed `(stage_id, task_number)`. execute_task_registry: Mutex, - /// `(stage_id, partition)` streams this proc's consumers abandoned, learned from inbound + /// `(stage_id, task_id, partition)` streams this proc's consumers abandoned, learned from inbound /// `Cancel` frames. A producer blocked on a full outbound checks it in its send spin and ends /// that stream cleanly, so a consumer that stopped reading early doesn't leave it spinning to /// the statement timeout. - cancelled_streams: Mutex>, + cancelled_streams: Mutex>, } #[derive(Default)] @@ -1755,22 +1809,16 @@ impl DrainHandle { } } - /// Record a `Cancel` frame: the consumer abandoned the `(stage_id, partition)` stream, so this - /// proc's producer of it stops. Idempotent. - fn note_cancel(&self, stage_id: u32, partition: u32) { - self.cancelled_streams - .lock() - .unwrap() - .insert((stage_id, partition)); + /// Record a `Cancel` frame: the consumer abandoned this data stream, so this proc's producer + /// of it stops. Idempotent. + fn note_cancel(&self, stream: MppDataStreamKey) { + self.cancelled_streams.lock().unwrap().insert(stream); } - /// Whether a consumer cancelled the `(stage_id, partition)` stream. Read by the send spin to - /// end a producer's stream cleanly when the consumer stopped reading. - pub(super) fn stream_cancelled(&self, stage_id: u32, partition: u32) -> bool { - self.cancelled_streams - .lock() - .unwrap() - .contains(&(stage_id, partition)) + /// Whether a consumer cancelled this data stream. Read by the send spin to end a producer's + /// stream cleanly when the consumer stopped reading. + pub(super) fn stream_cancelled(&self, stream: MppDataStreamKey) -> bool { + self.cancelled_streams.lock().unwrap().contains(&stream) } /// Take the receiving end of the `TaskMetrics` frame stream. The embedder (the leader) @@ -1938,7 +1986,7 @@ impl DrainHandle { guard .map .iter() - .filter(|((sender_proc, _, _), _)| self.scope_for_sender(*sender_proc) == scope) + .filter(|(key, _)| self.scope_for_sender(key.sender_proc) == scope) .map(|(_, buf)| buf.clone()) .collect::>() }; @@ -2025,15 +2073,14 @@ impl DrainHandle { Ok(rx) } - /// Register (or look up) the channel buffer for `(sender_proc, stage_id, partition)`. + /// Register (or look up) the channel buffer for one producer's data stream. /// The returned `Arc` is the canonical destination for frames matching /// that key: `try_drain_pass` pushes into the same entry on every `Batch { header, .. }` /// whose `header.sender_proc()` / `stage_id` / `partition` matches. - pub(super) fn register_channel( + pub(super) fn register_data_channel( &self, sender_proc: u32, - stage_id: u32, - partition: u32, + stream: MppDataStreamKey, ) -> Arc { let mut guard = self .channel_buffers @@ -2045,9 +2092,9 @@ impl DrainHandle { }; let buf = guard .map - .entry((sender_proc, stage_id, partition)) + .entry(PhysicalStreamKey::new(sender_proc, stream)) .or_insert_with(|| { - // num_sources stays 1: each (sender_proc, stage, partition) tuple has + // num_sources stays 1: each (sender_proc, stage, task, partition) tuple has // exactly one upstream (the named sender), even though the underlying // inbox is shared across all senders. DrainBuffer::new(1) @@ -2083,7 +2130,7 @@ impl DrainHandle { } } - /// Pull batches from each live receiver and demux them into the per-`(stage_id, partition)` + /// Pull batches from each live receiver and demux them into the per-`(stage_id, task_id, partition)` /// channel buffer registry. Called from `DrainGatherStream::poll_next` and from /// `MppSender::send_batch`'s cooperative spin. Drain work happens on the backend thread /// (pgrx-safe). No-op for thread-backed handles. @@ -2100,7 +2147,8 @@ impl DrainHandle { /// /// Routing rules per outcome: /// - `Batch { header, batch }`: look up (or lazily create) the - /// `(header.stage_id, header.partition)` channel buffer and push `batch`. + /// `(header.sender_proc(), header.stage_id, header.task_id, header.partition)` channel + /// buffer and push `batch`. /// - `Eof { header }`: per-channel EOF. Resolve the channel buffer and call /// `notify_source_done`. Other channels on the same queue keep flowing, /// so the receiver slot stays live. @@ -2122,19 +2170,13 @@ impl DrainHandle { for _ in 0..MAX_BATCHES_PER_SOURCE_PER_PASS { match rx.try_recv_batch() { RecvBatchOutcome::Batch { header, batch } => { - let buf = self.register_channel( - header.sender_proc(), - header.stage_id, - header.partition, - ); + let buf = + self.register_data_channel(header.sender_proc(), header.data_stream()); buf.push_batch(batch); } RecvBatchOutcome::Eof { header } => { - let buf = self.register_channel( - header.sender_proc(), - header.stage_id, - header.partition, - ); + let buf = + self.register_data_channel(header.sender_proc(), header.data_stream()); buf.notify_source_done(); // Other channels may still flow on this queue, so the receiver slot // stays live. @@ -2159,7 +2201,7 @@ impl DrainHandle { self.route_execute_task(header.stage_id, header.partition, frame); } RecvBatchOutcome::Cancel { header } => { - self.note_cancel(header.stage_id, header.partition); + self.note_cancel(header.data_stream()); } RecvBatchOutcome::Empty => break, RecvBatchOutcome::Detached => { @@ -2342,10 +2384,13 @@ mod tests { } impl MppSender { - /// Construct a sender with the default `(stage_id=0, partition=0)` header. Used where + /// Construct a sender with the default `(stage_id=0, task_id=0, partition=0)` header. Used where /// the header carries no actionable routing info. fn new(channel: Arc) -> Self { - Self::with_header(channel, MppFrameHeader::batch(0, 0, 0)) + Self::with_header( + channel, + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), + ) } /// Stats-less wrapper around `send_batch_traced`. Production call sites @@ -2507,7 +2552,7 @@ mod tests { #[test] fn frame_round_trips_a_batch_with_header() { let orig = sample_batch(64); - let header = MppFrameHeader::batch(7, 3, 0); + let header = MppFrameHeader::batch(MppDataStreamKey::new(7, 0, 3), 0); let mut buf = Vec::with_capacity(1024); encode_frame_into(header, &orig, &mut buf).expect("encode_frame"); @@ -2528,18 +2573,21 @@ mod tests { #[test] fn frame_round_trips_eof() { let mut buf = Vec::new(); - encode_eof_frame_into(2, 5, 0, &mut buf).expect("encode_eof"); + encode_eof_frame_into(MppDataStreamKey::new(2, 0, 5), 0, &mut buf).expect("encode_eof"); assert_eq!(buf.len(), MPP_FRAME_HEADER_SIZE); let (header, body) = decode_frame(&buf).expect("decode_frame"); - assert_eq!(header, MppFrameHeader::eof(2, 5, 0)); + assert_eq!( + header, + MppFrameHeader::eof(MppDataStreamKey::new(2, 0, 5), 0) + ); assert_eq!(header.kind().unwrap(), MppFrameKind::Eof); assert!(matches!(body, FrameBody::Eof)); } #[test] fn frame_round_trips_cancel() { - let header = MppFrameHeader::cancel(4, 2, 0); + let header = MppFrameHeader::cancel(MppDataStreamKey::new(4, 0, 2), 0); let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; header.write_to(&mut buf); @@ -2554,17 +2602,35 @@ mod tests { #[test] fn drain_records_cancel_scoped_to_its_stream() { let drain = DrainHandle::cooperative(0, Vec::new()); - assert!(!drain.stream_cancelled(7, 1)); - drain.note_cancel(7, 1); - assert!(drain.stream_cancelled(7, 1)); - // A cancel for one stream leaves the others alive, the way gRPC closes one stream: same - // stage but a different partition, and a different stage, both stay live. - assert!(!drain.stream_cancelled(7, 2)); - assert!(!drain.stream_cancelled(8, 1)); + let cancelled = MppDataStreamKey::new(7, 0, 1); + assert!(!drain.stream_cancelled(cancelled)); + drain.note_cancel(cancelled); + assert!(drain.stream_cancelled(cancelled)); + assert!(!drain.stream_cancelled(MppDataStreamKey::new(7, 3, 1))); + assert!(!drain.stream_cancelled(MppDataStreamKey::new(8, 0, 1))); + } + + #[tokio::test(flavor = "current_thread")] + async fn control_sender_ignores_matching_data_stream_cancel() { + let drain = Arc::new(DrainHandle::cooperative(0, Vec::new())); + // Control frames encode their target task in `partition`, so task 1 overlaps data + // `(stage 7, task 0, partition 1)`. + drain.note_cancel(MppDataStreamKey::new(7, 0, 1)); + let (tx, _rx) = in_proc_channel(1); + let sender = MppSender::with_header(Arc::new(tx), MppFrameHeader::task_metrics(7, 1, 0)) + .with_cooperative_drain(drain as Arc); + + let mut stats = SendBatchStats::default(); + assert!( + sender + .spin_send_frame(&[0xCA, 0xFE], &mut stats) + .await + .expect("control frame must not observe data cancellation") + ); } /// The producer-side half of the early-termination fix: a producer blocked on a full outbound - /// ends its send cleanly once a `Cancel` for its `(stage, partition)` stream lands on its inbox. + /// ends its send cleanly once a `Cancel` for its `(stage, task, partition)` stream lands on its inbox. /// The test hangs if the spin doesn't observe the cancel. #[tokio::test(flavor = "current_thread")] async fn producer_send_ends_when_consumer_cancels_the_stream() { @@ -2577,13 +2643,16 @@ mod tests { vec![(ReceiverScope::Inbox, MppReceiver::new(Box::new(inbox_rx)))], )); // Producer of the `(stage 7, partition 0)` stream. - let sender = MppSender::with_header(Arc::new(out_tx), MppFrameHeader::batch(7, 0, 1)) - .with_cooperative_drain(Arc::clone(&drain) as Arc); + let sender = MppSender::with_header( + Arc::new(out_tx), + MppFrameHeader::batch(MppDataStreamKey::new(7, 0, 0), 1), + ) + .with_cooperative_drain(Arc::clone(&drain) as Arc); // The consumer abandons that stream (it stopped reading before EOF): a `Cancel` reaches the // inbox. let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; - MppFrameHeader::cancel(7, 0, 0).write_to(&mut buf); + MppFrameHeader::cancel(MppDataStreamKey::new(7, 0, 0), 0).write_to(&mut buf); inbox_tx.send_bytes(&buf).unwrap(); let mut stats = SendBatchStats::default(); @@ -2600,6 +2669,56 @@ mod tests { .unwrap(); } + /// Tasks 0 and 3 can share one worker and output partition. + #[tokio::test(flavor = "current_thread")] + async fn cancelling_one_task_does_not_stop_sibling_sender() { + let (out_tx, out_rx) = in_proc_channel(4); + let out_tx: Arc = Arc::new(out_tx); + let (inbox_tx, inbox_rx) = in_proc_channel(4); + let drain = Arc::new(DrainHandle::cooperative( + 1, + vec![(ReceiverScope::Inbox, MppReceiver::new(Box::new(inbox_rx)))], + )); + let task_0_stream = MppDataStreamKey::new(7, 0, 0); + let task_3_stream = MppDataStreamKey::new(7, 3, 0); + let task_0 = + MppSender::with_header(Arc::clone(&out_tx), MppFrameHeader::batch(task_0_stream, 1)) + .with_cooperative_drain(Arc::clone(&drain) as Arc); + let task_3 = + MppSender::with_header(Arc::clone(&out_tx), MppFrameHeader::batch(task_3_stream, 1)) + .with_cooperative_drain(Arc::clone(&drain) as Arc); + + let mut cancel = vec![0u8; MPP_FRAME_HEADER_SIZE]; + MppFrameHeader::cancel(task_0_stream, 0).write_to(&mut cancel); + inbox_tx.send_bytes(&cancel).unwrap(); + drain.try_drain_pass().unwrap(); + assert!(task_0.stream_cancelled()); + assert!(!task_3.stream_cancelled()); + + let mut stats = SendBatchStats::default(); + task_0 + .send_batch_traced(&sample_batch(2), &mut stats) + .await + .unwrap(); + task_3 + .send_batch_traced(&sample_batch(3), &mut stats) + .await + .unwrap(); + task_3.send_eof_traced(&mut stats).await.unwrap(); + + let receiver = MppReceiver::new(Box::new(out_rx)); + let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { + panic!("task 3 batch must be delivered"); + }; + assert_eq!(header.data_stream(), task_3_stream); + assert_eq!(batch.num_rows(), 3); + let RecvBatchOutcome::Eof { header } = receiver.try_recv_batch() else { + panic!("task 3 EOF must be delivered"); + }; + assert_eq!(header.data_stream(), task_3_stream); + assert!(matches!(receiver.try_recv_batch(), RecvBatchOutcome::Empty)); + } + #[test] fn frame_round_trips_a_work_unit_and_feed_eof() { let unit = pb::WorkUnit { @@ -2796,6 +2915,7 @@ mod tests { magic: MPP_FRAME_MAGIC, flags: 0x42, // unknown kind byte, no reserved bits set stage_id: 0, + task_id: 0, partition: 0, }; let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; @@ -2813,6 +2933,7 @@ mod tests { magic: MPP_FRAME_MAGIC, flags: bit, // kind byte 0 (Batch), reserved bit set, no sender_proc stage_id: 0, + task_id: 0, partition: 0, }; let mut buf = vec![0u8; MPP_FRAME_HEADER_SIZE]; @@ -2834,6 +2955,7 @@ mod tests { magic: MPP_FRAME_MAGIC, flags: 0xFFFF_0001, // Eof in low byte, max sender_proc in high half, reserved=0 stage_id: 0, + task_id: 0, partition: 0, }; assert_eq!(header.kind().unwrap(), MppFrameKind::Eof); @@ -2844,7 +2966,8 @@ mod tests { fn frame_sender_proc_round_trip() { // sender_proc lives in flags bits 16..32 and shouldn't collide with kind or reserved. for &sp in &[0u32, 1, 7, 255, 256, 1023, 65534, MPP_MAX_SENDER_PROC] { - let header = MppFrameHeader::batch(11, 5, sp); + let stream = MppDataStreamKey::new(11, 0, 5); + let header = MppFrameHeader::batch(stream, sp); assert_eq!(header.sender_proc(), sp, "batch round-trip sp={sp}"); assert_eq!(header.kind().unwrap(), MppFrameKind::Batch); @@ -2855,7 +2978,7 @@ mod tests { assert_eq!(parsed.sender_proc(), sp, "decoded batch sender_proc"); let mut eof_buf = Vec::new(); - encode_eof_frame_into(11, 5, sp, &mut eof_buf).expect("encode eof"); + encode_eof_frame_into(stream, sp, &mut eof_buf).expect("encode eof"); let (parsed_eof, _) = decode_frame(&eof_buf).expect("decode eof"); assert_eq!(parsed_eof.sender_proc(), sp, "decoded eof sender_proc"); assert_eq!(parsed_eof.kind().unwrap(), MppFrameKind::Eof); @@ -2865,7 +2988,7 @@ mod tests { #[test] fn frame_eof_with_payload_is_rejected() { let mut buf = Vec::with_capacity(32); - encode_eof_frame_into(0, 0, 0, &mut buf).expect("encode_eof"); + encode_eof_frame_into(MppDataStreamKey::new(0, 0, 0), 0, &mut buf).expect("encode_eof"); buf.push(0xAB); // smuggle a payload byte after the Eof header let err = decode_frame(&buf).expect_err("Eof+payload must fail"); assert!(format!("{err}").contains("payload-less frame carries payload")); @@ -2876,7 +2999,12 @@ mod tests { let mut buf = Vec::with_capacity(1024); for rows in [0, 1, 7, 64, 1024] { let orig = sample_batch(rows); - encode_frame_into(MppFrameHeader::batch(0, 0, 0), &orig, &mut buf).expect("encode"); + encode_frame_into( + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), + &orig, + &mut buf, + ) + .expect("encode"); let (_header, body) = decode_frame(&buf).expect("decode"); let FrameBody::Batch(decoded) = body else { panic!("Batch frame must carry a batch payload"); @@ -2985,7 +3113,12 @@ mod tests { // learns to truncate sliced buffers, this assert flags the test (and the compact // step's main motivation) for review. let mut probe = Vec::new(); - encode_frame_into(MppFrameHeader::batch(3, 7, 0), &sliced, &mut probe).expect("probe"); + encode_frame_into( + MppFrameHeader::batch(MppDataStreamKey::new(3, 0, 7), 0), + &sliced, + &mut probe, + ) + .expect("probe"); assert!( probe.len() > cap, "premise: sliced encode ({} bytes) must exceed cap ({cap})", @@ -2995,7 +3128,7 @@ mod tests { let sink = BoundedSink::new(cap); let sender = MppSender::with_header( Arc::clone(&sink) as Arc, - MppFrameHeader::batch(3, 7, 0), + MppFrameHeader::batch(MppDataStreamKey::new(3, 0, 7), 0), ); sender.send_batch(&sliced).expect("chunked send"); @@ -3045,7 +3178,7 @@ mod tests { let sink = BoundedSink::new(cap); let sender = MppSender::with_header( Arc::clone(&sink) as Arc, - MppFrameHeader::batch(1, 1, 0), + MppFrameHeader::batch(MppDataStreamKey::new(1, 0, 1), 0), ); sender.send_batch(&sliced).expect("view-array chunked send"); @@ -3077,7 +3210,7 @@ mod tests { let sink = BoundedSink::new(cap); let sender = MppSender::with_header( Arc::clone(&sink) as Arc, - MppFrameHeader::batch(0, 0, 0), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), ); sender.send_batch(&batch).expect("giant row streams"); @@ -3096,58 +3229,80 @@ mod tests { assert_eq!(strings.value(0), value); } - /// Chunked frames from two streams of one proc may interleave on the shared inbox; the - /// per-stream reassembly keys keep them apart, and a complete frame passes through the - /// middle of an assembly untouched. + /// Chunked frames from two tasks on one proc may interleave on the shared inbox. Their stage + /// and output partition deliberately match, so only task identity keeps their reassembly + /// buffers apart. #[test] fn chunk_reassembly_keys_by_stream() { let sender_proc = 2; + let stream_a = MppDataStreamKey::new(1, 0, 1); + let stream_b = MppDataStreamKey::new(1, 3, 1); let inner_a = { let mut buf = Vec::new(); encode_frame_into( - MppFrameHeader::batch(1, 1, sender_proc), + MppFrameHeader::batch(stream_a, sender_proc), &sample_batch(4), &mut buf, ) .expect("encode inner A"); buf }; - let chunk_header = MppFrameHeader::chunk(1, 1, sender_proc); - let split_at = inner_a.len() / 2; + let inner_b = { + let mut buf = Vec::new(); + encode_frame_into( + MppFrameHeader::batch(stream_b, sender_proc), + &sample_batch(7), + &mut buf, + ) + .expect("encode inner B"); + buf + }; + let a_split = inner_a.len() / 2; + let b_split = inner_b.len() / 2; let mut a0 = Vec::new(); encode_chunk_frame_into( - chunk_header, + MppFrameHeader::chunk(stream_a, sender_proc), inner_a.len() as u64, 0, - &inner_a[..split_at], + &inner_a[..a_split], &mut a0, ); let mut a1 = Vec::new(); encode_chunk_frame_into( - chunk_header, + MppFrameHeader::chunk(stream_a, sender_proc), inner_a.len() as u64, - split_at as u64, - &inner_a[split_at..], + a_split as u64, + &inner_a[a_split..], &mut a1, ); - let mut b = Vec::new(); - encode_frame_into( - MppFrameHeader::batch(1, 2, sender_proc), - &sample_batch(7), - &mut b, - ) - .expect("encode B"); + let mut b0 = Vec::new(); + encode_chunk_frame_into( + MppFrameHeader::chunk(stream_b, sender_proc), + inner_b.len() as u64, + 0, + &inner_b[..b_split], + &mut b0, + ); + let mut b1 = Vec::new(); + encode_chunk_frame_into( + MppFrameHeader::chunk(stream_b, sender_proc), + inner_b.len() as u64, + b_split as u64, + &inner_b[b_split..], + &mut b1, + ); - let receiver = MppReceiver::new(ReplayChannel::over(vec![a0, b, a1])); - // B lands first: its complete frame doesn't wait on A's assembly. + let receiver = MppReceiver::new(ReplayChannel::over(vec![a0, b0, a1, b1])); let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { - panic!("expected B"); + panic!("expected reassembled task 0 batch"); }; - assert_eq!((header.partition, batch.num_rows()), (2, 7)); + assert_eq!(header.data_stream(), stream_a); + assert_eq!(batch.num_rows(), 4); let RecvBatchOutcome::Batch { header, batch } = receiver.try_recv_batch() else { - panic!("expected reassembled A"); + panic!("expected reassembled task 3 batch"); }; - assert_eq!((header.partition, batch.num_rows()), (1, 4)); + assert_eq!(header.data_stream(), stream_b); + assert_eq!(batch.num_rows(), 7); assert!(matches!(receiver.try_recv_batch(), RecvBatchOutcome::Empty)); } @@ -3158,12 +3313,12 @@ mod tests { let sender_proc = 3; let mut inner = Vec::new(); encode_frame_into( - MppFrameHeader::batch(5, 0, sender_proc), + MppFrameHeader::batch(MppDataStreamKey::new(5, 0, 0), sender_proc), &sample_batch(4), &mut inner, ) .expect("encode inner"); - let chunk_header = MppFrameHeader::chunk(5, 0, sender_proc); + let chunk_header = MppFrameHeader::chunk(MppDataStreamKey::new(5, 0, 0), sender_proc); let mut partial = Vec::new(); encode_chunk_frame_into( chunk_header, @@ -3173,7 +3328,8 @@ mod tests { &mut partial, ); let mut eof = Vec::new(); - encode_eof_frame_into(5, 0, sender_proc, &mut eof).expect("encode eof"); + encode_eof_frame_into(MppDataStreamKey::new(5, 0, 0), sender_proc, &mut eof) + .expect("encode eof"); // A fresh frame for the stream after the abandonment restarts cleanly at offset 0. let mut whole = Vec::new(); encode_chunk_frame_into(chunk_header, inner.len() as u64, 0, &inner, &mut whole); @@ -3198,7 +3354,7 @@ mod tests { #[test] fn chunk_out_of_sequence_errors() { let sender_proc = 4; - let chunk_header = MppFrameHeader::chunk(1, 0, sender_proc); + let chunk_header = MppFrameHeader::chunk(MppDataStreamKey::new(1, 0, 0), sender_proc); let mut orphan = Vec::new(); encode_chunk_frame_into(chunk_header, 1000, 500, &[0u8; 16], &mut orphan); let receiver = MppReceiver::new(ReplayChannel::over(vec![orphan])); @@ -3489,8 +3645,12 @@ mod tests { let mut enc_bytes = 0usize; let mut enc_buf = Vec::with_capacity(1024); for _ in 0..batches { - encode_frame_into(MppFrameHeader::batch(0, 0, 0), &template, &mut enc_buf) - .expect("encode"); + encode_frame_into( + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), + &template, + &mut enc_buf, + ) + .expect("encode"); enc_bytes += enc_buf.len(); } let enc_elapsed = enc_start.elapsed(); @@ -3573,11 +3733,13 @@ mod tests { } // --------------------------------------------------------------------- - // Per-`(stage_id, partition)` channel buffer registry on the cooperative `DrainHandle`. + // Per-`(stage_id, task_id, partition)` channel buffer registry on the cooperative `DrainHandle`. // - // Producers stamp `MppFrameHeader::batch(stage_id, partition)` on every outgoing frame, and + // Producers stamp `MppFrameHeader::batch(MppDataStreamKey::new(stage_id, task_id, + // partition), sender)` on + // every outgoing frame, and // the receiver-side cooperative drain demuxes by header into a channel buffer per - // `(stage_id, partition)`. These tests use the `in_proc_channel` backend to drive + // `(stage_id, task_id, partition)`. These tests use the `in_proc_channel` backend to drive // `try_drain_pass` from the test thread. That mirrors how the production path runs the drain // inline from `DrainGatherStream::poll_next` on the backend thread. // --------------------------------------------------------------------- @@ -3601,8 +3763,8 @@ mod tests { // teardown-EOF contract. let (tx, rx) = in_proc_channel(8); let base = MppSender::new(Arc::new(tx)); - let s00 = base.clone_with_header(MppFrameHeader::batch(0, 0, 0)); - let s01 = base.clone_with_header(MppFrameHeader::batch(0, 1, 0)); + let s00 = base.clone_with_header(MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0)); + let s01 = base.clone_with_header(MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 1), 0)); let receiver = MppReceiver::new(Box::new(rx)); let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); @@ -3613,8 +3775,8 @@ mod tests { drop(s01); drop(base); - let buf00 = handle.register_channel(0, 0, 0); - let buf01 = handle.register_channel(0, 0, 1); + let buf00 = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 0)); + let buf01 = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 1)); drain_until_detached(&handle); @@ -3638,19 +3800,25 @@ mod tests { // runs `cancel_channel_buffers`. let (tx, rx) = in_proc_channel(8); let tx_arc: Arc = Arc::new(tx); - let s00 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 0, 0)); - let s01 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 1, 0)); + let s00 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), + ); + let s01 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 1), 0), + ); let receiver = MppReceiver::new(Box::new(rx)); let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); s00.send_batch(&sample_batch(4)).unwrap(); let mut eof_buf = Vec::new(); - encode_eof_frame_into(0, 0, 0, &mut eof_buf).unwrap(); + encode_eof_frame_into(MppDataStreamKey::new(0, 0, 0), 0, &mut eof_buf).unwrap(); tx_arc.send_bytes(&eof_buf).unwrap(); s01.send_batch(&sample_batch(6)).unwrap(); - let buf00 = handle.register_channel(0, 0, 0); - let buf01 = handle.register_channel(0, 0, 1); + let buf00 = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 0)); + let buf01 = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 1)); drop(s00); drop(s01); @@ -3672,6 +3840,51 @@ mod tests { assert!(matches!(buf01.try_pop(), Some(DrainItem::Eof))); } + #[test] + fn drain_handle_keeps_same_worker_tasks_separate_after_one_eof() { + // A short PostgreSQL launch can place task 0 and task 3 on the same producer proc. + // They may both emit stage 7 / output partition 0, but task 0's EOF must not close task + // 3's stream. This is the exact failure mode that task_id in the frame key prevents. + let (tx, rx) = in_proc_channel(8); + let tx_arc: Arc = Arc::new(tx); + let task_0 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(7, 0, 0), 1), + ); + let task_3 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(7, 3, 0), 1), + ); + let receiver = MppReceiver::new(Box::new(rx)); + let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + task_0.send_batch(&sample_batch(2)).unwrap(); + task_3.send_batch(&sample_batch(3)).unwrap(); + let mut eof = Vec::new(); + encode_eof_frame_into(MppDataStreamKey::new(7, 0, 0), 1, &mut eof).unwrap(); + tx_arc.send_bytes(&eof).unwrap(); + task_3.send_batch(&sample_batch(5)).unwrap(); + encode_eof_frame_into(MppDataStreamKey::new(7, 3, 0), 1, &mut eof).unwrap(); + tx_arc.send_bytes(&eof).unwrap(); + + let task_0_buf = handle.register_data_channel(1, MppDataStreamKey::new(7, 0, 0)); + let task_3_buf = handle.register_data_channel(1, MppDataStreamKey::new(7, 3, 0)); + drain_until_detached(&handle); + + match task_0_buf.try_pop() { + Some(DrainItem::Batch(batch)) => assert_eq!(batch.num_rows(), 2), + other => panic!("expected task 0 batch, got {other:?}"), + } + assert!(matches!(task_0_buf.try_pop(), Some(DrainItem::Eof))); + + let mut task_3_rows = Vec::new(); + while let Some(DrainItem::Batch(batch)) = task_3_buf.try_pop() { + task_3_rows.push(batch.num_rows()); + } + assert_eq!(task_3_rows, vec![3, 5]); + assert!(matches!(task_3_buf.try_pop(), Some(DrainItem::Eof))); + } + #[test] fn drain_handle_register_channel_is_idempotent() { // Two calls for the same key return Arcs pointing to the same @@ -3680,8 +3893,9 @@ mod tests { let receiver = MppReceiver::new(Box::new(rx)); let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); - let first = handle.register_channel(0, 2, 3); - let second = handle.register_channel(0, 2, 3); + let stream = MppDataStreamKey::new(2, 0, 3); + let first = handle.register_data_channel(0, stream); + let second = handle.register_data_channel(0, stream); assert!(Arc::ptr_eq(&first, &second)); } @@ -3691,8 +3905,14 @@ mod tests { // The registry's compound key keeps them on separate channel buffers. let (tx, rx) = in_proc_channel(8); let tx_arc: Arc = Arc::new(tx); - let s_stage0 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(0, 0, 0)); - let s_stage1 = MppSender::with_header(Arc::clone(&tx_arc), MppFrameHeader::batch(1, 0, 0)); + let s_stage0 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(0, 0, 0), 0), + ); + let s_stage1 = MppSender::with_header( + Arc::clone(&tx_arc), + MppFrameHeader::batch(MppDataStreamKey::new(1, 0, 0), 0), + ); let receiver = MppReceiver::new(Box::new(rx)); let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); @@ -3703,8 +3923,8 @@ mod tests { drop(s_stage1); drop(tx_arc); - let buf0 = handle.register_channel(0, 0, 0); - let buf1 = handle.register_channel(0, 1, 0); + let buf0 = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 0)); + let buf1 = handle.register_data_channel(0, MppDataStreamKey::new(1, 0, 0)); drain_until_detached(&handle); @@ -3729,8 +3949,8 @@ mod tests { let receiver = MppReceiver::new(Box::new(rx)); let handle = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); - let buf_a = handle.register_channel(0, 0, 0); - let buf_b = handle.register_channel(0, 7, 3); + let buf_a = handle.register_data_channel(0, MppDataStreamKey::new(0, 0, 0)); + let buf_b = handle.register_data_channel(0, MppDataStreamKey::new(7, 0, 3)); // No data ever flows; the handle is just dropped. drop(handle); From d35973d7925adba1ea29be66c6eff8636c513a73 Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 9 Aug 2026 18:49:40 -0700 Subject: [PATCH 13/15] fix(shm): kept the control plane alive across a peer's clean detach. (#73) ## Ticket(s) Closed - Closes #70 ## What This PR keeps the control-plane registries and the drain slot alive when a ring's data senders drop. ## Why A peer that completes normally drops the ring's last data sender, and the drain then failed the whole inbox scope: unfinished channels plus the feed, plan, and request registries. The surviving worker may still be waiting on leader-origin frames (`ExecuteTask` requests, dispatched plans), so a clean completion killed independent work. That is the `joinscan_concurrent` flake in CI ([community](https://github.com/paradedb/paradedb/actions/runs/31221693830/job/93007528649), [enterprise](https://github.com/paradedb/paradedb-enterprise/actions/runs/31215016161/job/92986410089)). A detach only says the data plane is gone; control senders never join `sender_count` and can still publish. ## How The `Detached` arm now calls `detach_data_scope`: fail only the channels still awaiting their `Eof`, keep the registries and the receiver slot. The full `fail_scope` stays for the `Error` arm and teardown, so unexpected producer loss still fails dependent work. ## Tests - `peer_completion_detach_keeps_leader_control_plane_alive`: fails without the fix. - `producer_loss_fails_channels_still_awaiting_eof`: checks the loss path. --- src/shm/transport.rs | 197 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 179 insertions(+), 18 deletions(-) diff --git a/src/shm/transport.rs b/src/shm/transport.rs index 23e7c9215..9b304b00f 100644 --- a/src/shm/transport.rs +++ b/src/shm/transport.rs @@ -1970,19 +1970,29 @@ impl DrainHandle { } } - /// Fail every registered channel fed by `scope` that has not completed yet, and remember the - /// scope as dead so later registrations fail too. Channels whose `Eof` already arrived are - /// untouched: a detach after a clean drain is the normal end of life for a ring. - fn fail_scope(&self, scope: ReceiverScope, reason: &str) { + /// Mark `scope`'s data plane as departed: fail every registered channel still waiting on + /// its `Eof`, and remember the scope so later data registrations fail immediately. + /// Channels whose `Eof` already arrived are untouched: a detach after a clean drain is the + /// normal end of life for a ring. Idempotent. + /// + /// The feed, plan, and request registries stay live. Their frames come from control + /// senders, which never join a ring's `sender_count`, so "the last data sender dropped" + /// says nothing about them: a peer that completed normally has EOF'd everything it served, + /// while this proc may still owe the leader work it has not been asked for yet. + fn detach_data_scope(&self, scope: ReceiverScope, reason: &str) { let to_fail = { let mut guard = self .channel_buffers .lock() .expect("DrainHandle channel_buffers mutex poisoned"); - match scope { - ReceiverScope::Inbox => guard.dead_inbox = true, - ReceiverScope::SelfLoop => guard.dead_self_loop = true, + let dead = match scope { + ReceiverScope::Inbox => &mut guard.dead_inbox, + ReceiverScope::SelfLoop => &mut guard.dead_self_loop, + }; + if *dead { + return; } + *dead = true; guard .map .iter() @@ -1993,6 +2003,13 @@ impl DrainHandle { for buf in to_fail { buf.fail_pending(reason); } + } + + /// Hard scope death: [`Self::detach_data_scope`] plus the control-plane registries. For + /// ring corruption and embedder-driven teardown, where no further frame of any kind can be + /// trusted to arrive. + fn fail_scope(&self, scope: ReceiverScope, reason: &str) { + self.detach_data_scope(scope, reason); if scope == ReceiverScope::Inbox { let mut registry = self.feed_registry.lock().unwrap(); registry.dead = Some(reason.to_string()); @@ -2152,8 +2169,10 @@ impl DrainHandle { /// - `Eof { header }`: per-channel EOF. Resolve the channel buffer and call /// `notify_source_done`. Other channels on the same queue keep flowing, /// so the receiver slot stays live. - /// - `Detached` / `Error`: queue-wide shutdown. Notify every registered - /// channel buffer, mark the handle detached, and drop the slot. + /// - `Detached`: the receiver's data senders are gone. Fail the channels still + /// waiting on their `Eof`; the slot stays live for control frames. + /// - `Error`: queue-wide shutdown. Notify every registered channel buffer and + /// the control registries, mark the handle detached, and drop the slot. pub fn try_drain_pass(&self) -> Result<(), DataFusionError> { // Bound per-source pulls per call. The upper limit exists to give // the caller a chance to re-try its own send between drains — @@ -2205,15 +2224,17 @@ impl DrainHandle { } RecvBatchOutcome::Empty => break, RecvBatchOutcome::Detached => { - // Only THIS receiver is dead. The drain holds multiple receivers - // (own-inbox MPSC + self-loop in-proc); one going away doesn't - // imply the others have. Fail only the channels this receiver's - // scope feeds, and only those still waiting on their `Eof`: after a - // clean drain the detach is the ring's normal end of life, but a - // channel that never got its `Eof` (producer crash, early sender - // drop) would otherwise spin on `try_pop -> None` forever. - *slot = None; - self.fail_scope( + // Every data sender to this receiver is gone; that is also how a + // peer's normal completion looks. Its EOFs were drained ahead of the + // detach (the ring is FIFO), so failing only the channels still + // waiting on their `Eof` hits exactly the work that depended on a + // departed producer, which would otherwise spin on + // `try_pop -> None` forever. Control senders stay out of the ring's + // sender count and can still publish, so the slot stays live for the + // leader-origin frames a fragment may still be waiting on (requests, + // plans, feed units); `detach_data_scope` is idempotent, so the + // passes that keep observing the latch cost one failed recv each. + self.detach_data_scope( scope, "transport receiver detached before this channel's EOF; the \ producer went away", @@ -2889,6 +2910,146 @@ mod tests { assert!(format!("{err}").contains("producer went away")); } + /// Ring-header-aligned heap region standing in for the DSM segment. Returned first from + /// [`detached_test_inbox`] so it outlives every ring handle minted over it. + struct AlignedRingRegion { + ptr: *mut u8, + layout: std::alloc::Layout, + } + + impl AlignedRingRegion { + fn new(bytes: usize) -> Self { + let align = std::mem::align_of::(); + let layout = std::alloc::Layout::from_size_align(bytes, align).expect("layout"); + let ptr = unsafe { std::alloc::alloc_zeroed(layout) }; + assert!(!ptr.is_null()); + Self { ptr, layout } + } + } + + impl Drop for AlignedRingRegion { + fn drop(&mut self) { + unsafe { std::alloc::dealloc(self.ptr, self.layout) }; + } + } + + use crate::shm::Wakeup; + + struct NoWake; + impl Wakeup for NoWake { + fn wake(&self, _token: u64) {} + } + + /// One DSM test ring playing a worker's inbox: `peer` holds the ring's only data sender, + /// `leader` a control sender that stays out of `sender_count`. Dropping `peer` latches the + /// ring detached while `leader` can still publish, the exact state a peer's normal + /// completion leaves behind in a leader + two-worker mesh. + fn detached_test_inbox() -> ( + AlignedRingRegion, + super::super::mpsc_ring::DsmMpscSender, + super::super::mpsc_ring::DsmMpscSender, + DrainHandle, + ) { + use super::super::mesh::DsmInboxReceiver; + use super::super::mpsc_ring::{self, DsmMpscReceiver, DsmMpscRingHeader, DsmMpscSender}; + + let (ring_slots, slot_capacity) = (64u32, 1024u32); + let region = + AlignedRingRegion::new(DsmMpscRingHeader::region_bytes(ring_slots, slot_capacity)); + let header = unsafe { mpsc_ring::create_at(region.ptr, ring_slots, slot_capacity) }; + let nn = std::ptr::NonNull::new(header).expect("create_at returned null"); + let alive: super::super::AliveFlag = StdArc::new(std::sync::atomic::AtomicBool::new(true)); + let wakeup: StdArc = StdArc::new(NoWake); + let peer = unsafe { DsmMpscSender::new(nn, StdArc::clone(&wakeup), StdArc::clone(&alive)) }; + let leader = unsafe { + DsmMpscSender::new_control(nn, StdArc::clone(&wakeup), StdArc::clone(&alive)) + }; + let receiver = unsafe { DsmMpscReceiver::new(nn, alive) }; + let drain = DrainHandle::cooperative( + 2, + vec![( + ReceiverScope::Inbox, + MppReceiver::new(Box::new(DsmInboxReceiver::new(receiver))), + )], + ); + (region, peer, leader, drain) + } + + /// A peer's normal completion drops the ring's last data sender. Channels the peer EOF'd + /// stay clean, parked request takers keep waiting, and a control frame published after + /// the detach still reaches its registry: this proc may still owe the leader work. + #[test] + fn peer_completion_detach_keeps_leader_control_plane_alive() { + let (_region, peer, leader, drain) = detached_test_inbox(); + + // A fragment mid-query holds a parked request taker. + let mut requests = drain.take_execute_task_rx(3, 1).expect("take rx"); + + // The peer serves its one stream to completion and departs. + let mut eof = Vec::new(); + encode_eof_frame_into(MppDataStreamKey::new(2, 1, 0), 1, &mut eof).expect("eof frame"); + peer.try_send(&eof).expect("eof send"); + drop(peer); + + drain.try_drain_pass().expect("drain over the detach"); + + // The served stream completed; the departure did not fail it. + let served = drain.register_data_channel(1, MppDataStreamKey::new(2, 1, 0)); + assert!(matches!(served.try_pop(), Some(DrainItem::Eof))); + + // The request taker is still parked, not failed. + assert!(matches!( + requests.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + + // A request the leader publishes after the detach still lands. + let request = ExecuteTaskFrame::from_parts( + pb::ExecuteTaskRequest { + task_key: Some(pb::TaskKey { + query_id: vec![7; 16], + stage_id: 3, + task_number: 1, + }), + target_partition_start: 0, + target_partition_end: 1, + producer_head: Some(pb::execute_task_request::ProducerHead::None( + pb::NoneHead {}, + )), + }, + &http::HeaderMap::new(), + ) + .expect("request frame"); + let mut frame = Vec::new(); + encode_prost_frame_into(MppFrameHeader::execute_task(3, 1, 0), &request, &mut frame) + .expect("encode request"); + leader.try_send(&frame).expect("post-detach control send"); + drain.try_drain_pass().expect("drain the request"); + + let got = requests + .try_recv() + .expect("request frame must be delivered") + .expect("delivery must not be a failure"); + let (decoded, _headers) = got.into_parts().expect("into_parts"); + assert_eq!(decoded.task_key.unwrap().task_number, 1); + } + + /// A producer that departs without a channel's `Eof` still fails that channel: the + /// consumer depends on data that will never arrive. Registrations after the detach fail + /// the same way. + #[test] + fn producer_loss_fails_channels_still_awaiting_eof() { + let (_region, peer, _leader, drain) = detached_test_inbox(); + + let dependent = drain.register_data_channel(1, MppDataStreamKey::new(2, 1, 0)); + drop(peer); + drain.try_drain_pass().expect("drain over the detach"); + + assert!(matches!(dependent.try_pop(), Some(DrainItem::Failed(_)))); + let late = drain.register_data_channel(1, MppDataStreamKey::new(2, 1, 1)); + assert!(matches!(late.try_pop(), Some(DrainItem::Failed(_)))); + } + #[test] fn frame_rejects_short_message() { let too_short = vec![0u8; MPP_FRAME_HEADER_SIZE - 1]; From 08660203082a6228326a2cfce935651ed10a8d63 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Mon, 10 Aug 2026 12:08:04 +0900 Subject: [PATCH 14/15] Add support for explicitly sending errors in the shared memory implementation. (#66) Adds an explicit `TaskError` message type, and ensures that task errors and panics are sent across the wire. In order to do that, moves to better encapsulate the `run_execute_task_loop` method. --- src/shm/README.md | 2 +- src/shm/in_process.rs | 119 +++++++++++++++++++++++++++++-- src/shm/runtime.rs | 27 ++++++- src/shm/self_hosted.rs | 11 +-- src/shm/setup.rs | 137 ++++++++++++++++++++++++++++------- src/shm/transport.rs | 157 +++++++++++++++++++++++++++++++++++++---- 6 files changed, 395 insertions(+), 58 deletions(-) diff --git a/src/shm/README.md b/src/shm/README.md index 4dd64826b..1e81feda3 100644 --- a/src/shm/README.md +++ b/src/shm/README.md @@ -35,7 +35,7 @@ The shared-memory transport mirrors the canonical gRPC protocol's pull-based RPC - **Demand-Driven Pull Execution**: - Downstream consumers issue `ExecuteTaskFrame` to upstream producer tasks via `MppMesh::send_execute_task`. - Upstream worker tasks wait on `MppMesh::take_execute_task_rx(stage_num, task_idx)` for incoming execution requests before spawning task fragments. - - Workers run the demand loop via [`run_execute_task_loop`](./setup.rs), which validates requested partition ranges (`start..end`), guards against duplicate/overlapping partition claims, listens to `CancellationToken` for prompt cancellation unwinding, and periodically drives inbound ring draining. + - Workers run the demand loop via [`run_execute_task_loop`](./setup.rs), which internally waits for incoming `ExecuteTaskFrame` execution requests, handles routing `TaskError` frames back to the leader if a task panics, validates requested partition ranges (`start..end`), guards against duplicate/overlapping partition claims, listens to `CancellationToken` for prompt cancellation unwinding, and periodically drives inbound ring draining. - **Demuxing**: Incoming frames are demuxed cooperatively by [`DrainHandle`](./transport.rs) into per-`(stage_id, task_number)` request registries and per-`(sender_proc, stage_id, task_id, partition)` record-batch buffers. ### 2. Data Plane (DSM Ring Buffers) diff --git a/src/shm/in_process.rs b/src/shm/in_process.rs index e4a8ecdb9..8b251cb4c 100644 --- a/src/shm/in_process.rs +++ b/src/shm/in_process.rs @@ -441,16 +441,12 @@ async fn run_worker_proc( let worker_sink = Arc::clone(&worker_sink); futures.push(async move { - let rx = mesh.take_execute_task_rx(stage_id, task_idx)?; - let mesh = Arc::clone(&mesh); crate::shm::setup::run_execute_task_loop( - rx, + &mesh, + stage_id, + task_idx, n_out, tokio_util::sync::CancellationToken::new(), - || { - mesh.inbound_receiver().try_drain_pass()?; - Ok(()) - }, |_request, _headers, range| { let plan = Arc::clone(&plan); let task_ctx = Arc::clone(&task_ctx); @@ -1247,4 +1243,113 @@ mod tests { "unexpected error: {err}" ); } + + /// When a task fragment panics on a worker process during execution, the worker loop catches + /// the panic and sends a `TaskError` frame to the requesting process. The requesting process's + /// drain handle marks the channel scope failed, ensuring the leader or caller query receives + /// an error containing the panic message rather than hanging on missing batches. + #[tokio::test(flavor = "current_thread")] + async fn in_process_fragment_panic_propagates_to_leader() { + use crate::proto as pb; + use crate::shm::run_execute_task_loop; + use crate::shm::transport::{DrainItem, ExecuteTaskFrame}; + use tokio_util::sync::CancellationToken; + + let n_procs = 2; // Proc 0: leader / requester, Proc 1: worker + let region_total = dsm_region_bytes(n_procs, IN_PROCESS_QUEUE_BYTES, 0).unwrap(); + let region = HeapRegion::new(region_total); + let base = SharedBase(region.base()); + let wakeup: Arc = Arc::new(NoopWakeup); + + let leader_mesh = unsafe { + leader_setup( + base.0, + n_procs, + IN_PROCESS_QUEUE_BYTES, + &[], + Arc::clone(&wakeup), + receiver_token(0), + Arc::new(NoInterrupt), + /* attach_senders */ true, + ) + } + .unwrap() + .mesh; + + let worker_attach = unsafe { + worker_setup( + base.0, + region_total, + 1, + Arc::clone(&wakeup), + receiver_token(1), + Arc::new(NoInterrupt), + ) + } + .unwrap(); + + use datafusion::common::runtime::SpawnedTask; + + let token = CancellationToken::new(); + + // Worker 1 runs execute task loop where the fragment panics. + let worker_mesh = Arc::clone(&worker_attach.mesh); + let loop_mesh = Arc::clone(&worker_mesh); + let worker_token = token.clone(); + let worker_handle = SpawnedTask::spawn(async move { + run_execute_task_loop( + &loop_mesh, + 1, // stage_id 1 + 0, // task_number 0 + 1, // 1 partition + worker_token, + |_req, _hdr, _range| async move { + panic!("simulated worker fragment crash"); + }, + ) + .await + }); + + // Leader (proc 0) registers a channel for task (stage 1, task 0, partition 0) and sends ExecuteTask. + let channel = leader_mesh + .inbound_receiver() + .register_data_channel(1, MppDataStreamKey::new(1, 0, 0)); + + let request = ExecuteTaskFrame::from_parts( + pb::ExecuteTaskRequest { + task_key: Some(pb::TaskKey { + query_id: vec![1; 16], + stage_id: 1, + task_number: 0, + }), + target_partition_start: 0, + target_partition_end: 1, + producer_head: None, + }, + &http::HeaderMap::new(), + ) + .unwrap(); + + leader_mesh.send_execute_task(1, 0, request).await.unwrap(); + + // Drive drain passes on worker and leader until the TaskError arrives. + let mut item = None; + for _ in 0..20 { + let _ = worker_mesh.inbound_receiver().try_drain_pass(); + let _ = leader_mesh.inbound_receiver().try_drain_pass(); + item = channel.try_pop(); + if item.is_some() { + break; + } + tokio::task::yield_now().await; + } + + let item = item.expect("TaskError frame must arrive and mark channel failed"); + assert!( + matches!(item, DrainItem::Failed(ref msg) if msg.contains("simulated worker fragment crash")), + "Expected DrainItem::Failed with panic message, got: {item:?}" + ); + + let _ = worker_handle.join().await; + } } diff --git a/src/shm/runtime.rs b/src/shm/runtime.rs index 3d85cb53f..21bb52b75 100644 --- a/src/shm/runtime.rs +++ b/src/shm/runtime.rs @@ -209,7 +209,7 @@ impl MppMesh { } /// Obtains the channel receiver for incoming `ExecuteTask` requests targeting `(stage_id, task_number)`. - pub fn take_execute_task_rx( + pub(crate) fn take_execute_task_rx( self: &Arc, stage_id: u32, task_number: u32, @@ -236,6 +236,31 @@ impl MppMesh { } } + /// Obtains an outbound sender pre-configured to route a `TaskError` frame back to `dest_proc` + /// (the process that requested task execution for `(stage_id, task_number)`). + /// + /// Returning `TaskError` to `dest_proc` matches standard RPC response routing: the node + /// that requested execution is directly notified of task failure or fragment panic so its + /// drain handle can fail dependent stream waiters. + pub(crate) fn error_sender( + &self, + dest_proc: u32, + stage_id: u32, + task_number: u32, + ) -> Option { + let guard = self.cancel_senders.lock().unwrap(); + let senders = guard.as_ref()?; + let senders = senders.lock().unwrap(); + let base = senders.get(dest_proc as usize)?.as_ref()?; + Some( + base.clone_with_header(crate::shm::transport::MppFrameHeader::task_error( + stage_id, + task_number, + self.this_proc, + )), + ) + } + /// The single cooperative inbound handle that pulls frames from every peer (and the /// self-loop) into per-`(sender_proc, stage_id, task_id, partition)` channel buffers. pub fn inbound_receiver(&self) -> &Arc { diff --git a/src/shm/self_hosted.rs b/src/shm/self_hosted.rs index b55ffce8e..4fe6223fa 100644 --- a/src/shm/self_hosted.rs +++ b/src/shm/self_hosted.rs @@ -896,17 +896,13 @@ impl TaskDriver { ); } - let rx = launch.mesh.take_execute_task_rx(stage_num, task_i as u32)?; let mut sinks_opt: Vec<_> = launch.sinks.into_iter().map(Some).collect(); - let mesh = Arc::clone(&launch.mesh); let produce = crate::shm::setup::run_execute_task_loop( - rx, + &launch.mesh, + stage_num, + task_i as u32, n_partitions, token.clone(), - || { - mesh.inbound_receiver().try_drain_pass()?; - Ok(()) - }, |request, _headers, range| { let len = range.len(); let mut task_sinks = Vec::with_capacity(len); @@ -971,7 +967,6 @@ impl TaskDriver { }, ); let produce_res: Result<()> = produce.await; - // The metrics receiver resolves as the last partition stream above completes, so this // does not block on anything remote. The frame goes back over the mesh, where the // leader-side pump forwards it into the metrics store. diff --git a/src/shm/setup.rs b/src/shm/setup.rs index 47832c7f0..fdb6d4283 100644 --- a/src/shm/setup.rs +++ b/src/shm/setup.rs @@ -45,8 +45,9 @@ use super::mesh::{DsmInboxReceiver, DsmInboxSender}; use super::mpsc_ring::{DsmMpscSender, NO_RECEIVER_TOKEN, Wakeup}; use super::runtime::MppMesh; use super::transport::{ - BatchChannelSender, DrainHandle, ExecuteTaskRx, Interrupt, MppDataStreamKey, MppFrameHeader, - MppReceiver, MppSender, ReceiverScope, SELF_LOOP_CAPACITY, in_proc_channel, + BatchChannelSender, DrainHandle, ExecuteTaskRx, IncomingExecuteTaskRequest, Interrupt, + MppDataStreamKey, MppFrameHeader, MppReceiver, MppSender, ReceiverScope, SELF_LOOP_CAPACITY, + in_proc_channel, }; use crate::proto as pb; use crate::work_unit_feed::RemoteWorkUnitFeedRegistry; @@ -400,7 +401,7 @@ pub async fn run_worker_fragment( /// ranges, and delegating execution of each requested range to `spawn_range`. /// /// # Lifecycle & Guarantees -/// - **On-Demand Range Execution:** Listens on `rx` for incoming [`ExecuteTaskFrame`] messages, +/// - **On-Demand Range Execution:** Listens on the mesh for incoming [`ExecuteTaskFrame`] messages, /// extracting the requested partition range (`start..end`) and headers. /// - **Partition Validation & Single-Claim Accounting:** Ensures that `start <= end` and `end <= n_partitions`, /// and verifies that no partition within `start..end` has been claimed by a previous frame. Returns an @@ -408,30 +409,61 @@ pub async fn run_worker_fragment( /// - **Cancellation & Early Termination Unwinding:** Selects on `token.cancelled()`. If the query is /// cancelled or terminates early (e.g. satisfied by a `LIMIT` clause downstream), the request loop /// breaks out immediately and drops active sub-futures. -/// - **Cooperative Inbound Ring Flushing:** Periodically invokes `drain_pass` to drive the inbound ring -/// buffer drain pass while awaiting frame arrivals or stream completions. -/// - **Completion:** Exits cleanly once all `n_partitions` have been requested (or `rx` closes) AND all +/// - **Cooperative Inbound Ring Flushing:** Periodically invokes the mesh's inbound receiver's drain pass +/// while awaiting frame arrivals or stream completions. +/// - **Completion:** Exits cleanly once all `n_partitions` have been requested (or the channel closes) AND all /// spawned stream futures in `spawn_range` have completed to EOF. /// -/// # Arguments -/// * `rx`: The receiver channel for incoming [`ExecuteTaskFrame`] messages obtained via `MppMesh::take_execute_task_rx`. +/// * `mesh`: The MPP mesh used to coordinate routing, draining, and error handling. +/// * `stage_id`: The stage ID this task belongs to. +/// * `task_number`: The task index within the stage. /// * `n_partitions`: Total number of output partitions expected across all request frames for this task. /// * `token`: Cancellation token used to unwind the request loop on query cancellation or early teardown. -/// * `drain_pass`: Callback invoked periodically to drive inbound ring buffer draining (e.g. `mesh.inbound_receiver().try_drain_pass()`). /// * `spawn_range`: Closure invoked for each valid, unclaimed partition range `start..end`. Must return a future executing the partition streams for that range. pub async fn run_execute_task_loop( + mesh: &Arc, + stage_id: u32, + task_number: u32, + n_partitions: usize, + token: CancellationToken, + spawn_range: F, +) -> Result<()> +where + F: FnMut(pb::ExecuteTaskRequest, http::HeaderMap, std::ops::Range) -> Fut, + Fut: Future> + Send + 'static, +{ + let rx = mesh.take_execute_task_rx(stage_id, task_number)?; + let drain_pass = { + let inbound = Arc::clone(mesh.inbound_receiver()); + move || inbound.try_drain_pass() + }; + let (res, failed_sender_proc) = + run_execute_task_loop_inner(rx, n_partitions, token, drain_pass, spawn_range).await; + // Only send TaskError if the loop returned an actual execution error or fragment panic. + // Healthy early termination (such as downstream LIMIT queries or cancellation token unwinding) + // completes cleanly and returns Ok(()), so no TaskError is published. + if let (Err(e), Some(sender)) = ( + &res, + failed_sender_proc.and_then(|p| mesh.error_sender(p, stage_id, task_number)), + ) { + let _ = sender.send_task_error_best_effort(&e.to_string()).await; + } + res +} + +async fn run_execute_task_loop_inner( rx: ExecuteTaskRx, n_partitions: usize, token: CancellationToken, mut drain_pass: impl FnMut() -> Result<()>, mut spawn_range: F, -) -> Result<()> +) -> (Result<()>, Option) where F: FnMut(pb::ExecuteTaskRequest, http::HeaderMap, std::ops::Range) -> Fut, Fut: Future> + Send + 'static, { if n_partitions == 0 { - return Ok(()); + return (Ok(()), None); } let mut sub_futures = futures::stream::FuturesUnordered::new(); @@ -452,37 +484,59 @@ where frame_res = async { rx_opt.as_mut().unwrap().recv().await }, if rx_opt.is_some() => { match frame_res { - Some(Ok(frame)) => { - let (request, headers) = frame.into_parts()?; + Some(Ok(IncomingExecuteTaskRequest { sender_proc, frame })) => { + let (request, headers) = match frame.into_parts() { + Ok(parts) => parts, + Err(e) => return (Err(e), Some(sender_proc)), + }; let start = request.target_partition_start as usize; let end = request.target_partition_end as usize; if start > end || end > n_partitions { - return Err(DataFusionError::Execution(format!( + return (Err(DataFusionError::Execution(format!( "shm transport: invalid partition range {start}..{end} for total partitions {n_partitions}" - ))); + ))), Some(sender_proc)); } for (i, claimed_slot) in claimed[start..end].iter_mut().enumerate() { if *claimed_slot { let q = start + i; - return Err(DataFusionError::Execution(format!( + return (Err(DataFusionError::Execution(format!( "shm transport: requested partition range {start}..{end} includes already claimed partition {q}" - ))); + ))), Some(sender_proc)); } *claimed_slot = true; } - let len = end - start; let range_fut = spawn_range(request, headers, start..end); - sub_futures.push(range_fut); + sub_futures.push(async move { + use futures::FutureExt; + let res = std::panic::AssertUnwindSafe(range_fut) + .catch_unwind() + .await; + match res { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err((sender_proc, e)), + Err(payload) => { + let msg = if let Some(s) = payload.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + "Box".to_string() + }; + Err((sender_proc, DataFusionError::Execution(msg))) + } + } + }); + let len = end - start; partitions_requested += len; if partitions_requested >= n_partitions { rx_opt = None; } } - Some(Err(e)) => return Err(e), + Some(Err(e)) => return (Err(e), None), None => { rx_opt = None; } @@ -491,16 +545,18 @@ where next_res = sub_futures.next(), if !sub_futures.is_empty() => { match next_res { Some(Ok(())) => {} - Some(Err(e)) => return Err(e), + Some(Err((sender_proc, e))) => return (Err(e), Some(sender_proc)), None => unreachable!(), } } _ = tokio::time::sleep(std::time::Duration::from_millis(1)) => { - drain_pass()?; + if let Err(e) = drain_pass() { + return (Err(e), None); + } } } } - Ok(()) + (Ok(()), None) } #[cfg(test)] @@ -549,13 +605,21 @@ mod tests { ) .unwrap(); - tx.send(Ok(frame1)).unwrap(); - tx.send(Ok(frame2)).unwrap(); + tx.send(Ok(IncomingExecuteTaskRequest { + sender_proc: 0, + frame: frame1, + })) + .unwrap(); + tx.send(Ok(IncomingExecuteTaskRequest { + sender_proc: 0, + frame: frame2, + })) + .unwrap(); let ranges_counter = Arc::clone(&ranges_processed); let len_counter = Arc::clone(&total_partition_len_processed); - let res = run_execute_task_loop( + let (res, _) = run_execute_task_loop_inner( rx, total_partitions, token, @@ -577,4 +641,25 @@ mod tests { assert_eq!(ranges_processed.load(Ordering::SeqCst), 2); assert_eq!(total_partition_len_processed.load(Ordering::SeqCst), 4); } + + #[tokio::test] + async fn run_execute_task_loop_cancellation_returns_ok() { + let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let token = CancellationToken::new(); + token.cancel(); // Pre-cancelled token to simulate early query termination / LIMIT. + + let (res, _) = run_execute_task_loop_inner( + rx, + 4, + token, + || Ok(()), + |_request, _headers, _range| async { Ok(()) }, + ) + .await; + + assert!( + res.is_ok(), + "cancellation token unwinding must return Ok(()) to avoid false TaskError dispatch" + ); + } } diff --git a/src/shm/transport.rs b/src/shm/transport.rs index 9b304b00f..7cc7facc6 100644 --- a/src/shm/transport.rs +++ b/src/shm/transport.rs @@ -93,6 +93,10 @@ pub(super) enum MppFrameKind { /// exceeds the ring and frame size is never an error. Chunk = 7, ExecuteTask = 8, + /// Carries a worker's unrecoverable error (e.g. a panic or `DataFusionError`) string + /// to the consumer, so it sees the actual crash reason instead of a + /// detached channel. + TaskError = 9, } /// Payload of a `SetPlan` frame: the plan-delivery message a worker needs to run one task, @@ -368,6 +372,17 @@ impl MppFrameHeader { } } + /// Build a `TaskError` header destined for the consumer of `(stage_id, task_number)`. + pub fn task_error(stage_id: u32, partition: u32, sender_proc: u32) -> Self { + Self { + magic: MPP_FRAME_MAGIC, + flags: pack_flags(MppFrameKind::TaskError, sender_proc), + stage_id, + task_id: 0, + partition, + } + } + /// Build a `SetPlan` header for `(stage_id, task_number)`: the frame delivers that task's /// plan to the proc hosting it. pub fn set_plan(stage_id: u32, task_number: u32, sender_proc: u32) -> Self { @@ -450,6 +465,7 @@ impl MppFrameHeader { 6 => Ok(MppFrameKind::Cancel), 7 => Ok(MppFrameKind::Chunk), 8 => Ok(MppFrameKind::ExecuteTask), + 9 => Ok(MppFrameKind::TaskError), other => Err(DataFusionError::Internal(format!( "mpp: unknown frame kind {other:#x}" ))), @@ -658,6 +674,7 @@ enum FrameBody { SetPlan(SetPlanFrame), ExecuteTask(ExecuteTaskFrame), Cancel, + TaskError(String), } /// Inverse of the frame encoders. Parses the 20-byte header and decodes the payload according @@ -711,6 +728,10 @@ fn decode_frame(bytes: &[u8]) -> Result<(MppFrameHeader, FrameBody), DataFusionE "mpp: chunk frame reached decode_frame; chunks reassemble in MppReceiver first" .to_string(), )), + MppFrameKind::TaskError => { + let msg = String::from_utf8_lossy(payload).into_owned(); + Ok((header, FrameBody::TaskError(msg))) + } } } @@ -1378,6 +1399,47 @@ impl MppSender { self.scratch.replace(scratch); result } + + /// Like `send_task_metrics_best_effort`, but for sending a task's fatal error message. + /// Sent explicitly by the producer (via `run_execute_task_loop`) on a panic. + pub(crate) async fn send_task_error_best_effort( + &self, + msg: &str, + ) -> Result<(), DataFusionError> { + let mut scratch = self.scratch.replace(Vec::new()); + let result = async { + scratch.clear(); + scratch.resize(MPP_FRAME_HEADER_SIZE, 0); + let error_header = MppFrameHeader::task_error( + self.header.stage_id, + self.header.partition, + self.header.sender_proc(), + ); + error_header.write_to(&mut scratch[..MPP_FRAME_HEADER_SIZE]); + + let mut msg_bytes = msg.as_bytes(); + if let Some(cap) = self.channel.max_frame_bytes() { + let max_payload_len = cap.saturating_sub(MPP_FRAME_HEADER_SIZE); + if msg_bytes.len() > max_payload_len { + msg_bytes = &msg_bytes[..max_payload_len]; + } + } + scratch.extend_from_slice(msg_bytes); + + let _send_guard = self.channel.send_lock().lock().await; + for _ in 0..MAX_CONTROL_SEND_SPINS { + match self.channel.try_send_bytes(&scratch) { + Ok(true) => return Ok(()), + Ok(false) => tokio::task::yield_now().await, + Err(_) => return Ok(()), + } + } + Ok(()) + } + .await; + self.scratch.replace(scratch); + result + } } impl Clone for MppSender { @@ -1478,7 +1540,7 @@ impl MppReceiver { Ok(Some(frame)) => frame, // Absorbed one chunk of a larger frame; pull the next message. Ok(None) => continue, - Err(e) => return RecvBatchOutcome::Error(e), + Err(e) => return RecvBatchOutcome::TransportError(e), }; return match decode_frame(&complete) { Ok((header, FrameBody::Batch(batch))) => { @@ -1499,7 +1561,10 @@ impl MppReceiver { RecvBatchOutcome::ExecuteTask { header, frame } } Ok((header, FrameBody::Cancel)) => RecvBatchOutcome::Cancel { header }, - Err(e) => RecvBatchOutcome::Error(e), + Ok((header, FrameBody::TaskError(msg))) => { + RecvBatchOutcome::TaskError { header, msg } + } + Err(e) => RecvBatchOutcome::TransportError(e), }; } RecvOutcome::Empty => return RecvBatchOutcome::Empty, @@ -1625,7 +1690,15 @@ pub(super) enum RecvBatchOutcome { }, Empty, Detached, - Error(DataFusionError), + /// An explicitly transmitted error from a remote worker for a specific task (e.g. panic or query error). + /// The transport channel itself is perfectly healthy and may continue to yield frames. + TaskError { + header: MppFrameHeader, + msg: String, + }, + /// A fatal structural or protocol error on the local side (e.g. corrupt bytes, IPC + /// decode failure). The transport channel is unusable. + TransportError(DataFusionError), } /// Per-`(sender_proc, stage_id, task_id, partition)` channel buffer registry owned by a cooperative @@ -1772,8 +1845,14 @@ enum SetPlanSlot { Waiting(tokio::sync::oneshot::Sender>), } +#[derive(Clone, Debug, PartialEq)] +pub struct IncomingExecuteTaskRequest { + pub sender_proc: u32, + pub frame: ExecuteTaskFrame, +} + pub type ExecuteTaskRx = - tokio::sync::mpsc::UnboundedReceiver>; + tokio::sync::mpsc::UnboundedReceiver>; #[derive(Default)] struct ExecuteTaskRegistry { @@ -1782,8 +1861,8 @@ struct ExecuteTaskRegistry { } enum ExecuteTaskSlot { - Pending(Vec>), - Active(tokio::sync::mpsc::UnboundedSender>), + Pending(Vec>), + Active(tokio::sync::mpsc::UnboundedSender>), } impl DrainHandle { @@ -2040,6 +2119,7 @@ impl DrainHandle { &self, stage_id: u32, task_number: u32, + sender_proc: u32, frame: ExecuteTaskFrame, ) { let mut guard = self.execute_task_registry.lock().unwrap(); @@ -2050,10 +2130,11 @@ impl DrainHandle { .map .entry((stage_id, task_number)) .or_insert_with(|| ExecuteTaskSlot::Pending(Vec::new())); + let item = Ok(IncomingExecuteTaskRequest { sender_proc, frame }); match slot { - ExecuteTaskSlot::Pending(queue) => queue.push(Ok(frame)), + ExecuteTaskSlot::Pending(queue) => queue.push(item), ExecuteTaskSlot::Active(tx) => { - let _ = tx.send(Ok(frame)); + let _ = tx.send(item); } } } @@ -2217,11 +2298,28 @@ impl DrainHandle { self.route_set_plan(header.stage_id, header.partition, frame); } RecvBatchOutcome::ExecuteTask { header, frame } => { - self.route_execute_task(header.stage_id, header.partition, frame); + self.route_execute_task( + header.stage_id, + header.partition, + header.sender_proc(), + frame, + ); } RecvBatchOutcome::Cancel { header } => { self.note_cancel(header.data_stream()); } + RecvBatchOutcome::TaskError { header, msg } => { + // The remote worker sent a crash message, but the transport is intact. + // Fail the scope to unblock waiters, but leave this receiver in the slot: + // the worker will likely close the channel next, yielding `Detached`. + self.fail_scope( + scope, + &format!( + "remote task {}.{} failed: {}", + header.stage_id, header.partition, msg + ), + ); + } RecvBatchOutcome::Empty => break, RecvBatchOutcome::Detached => { // Every data sender to this receiver is gone; that is also how a @@ -2241,7 +2339,9 @@ impl DrainHandle { ); break; } - RecvBatchOutcome::Error(e) => { + RecvBatchOutcome::TransportError(e) => { + // The channel is corrupted or broken. Polling it again is dangerous. + // We evict this receiver (`*slot = None`) to prevent future polls. // Same scoping as Detached, but the ring reported corruption (or the // receiver poisoned itself), so even completed siblings can't be // trusted to have been the last word. Still scope-limited: the other @@ -2370,6 +2470,32 @@ mod tests { assert!(rx2.unwrap_err().to_string().contains("already taken")); } + #[tokio::test] + async fn send_task_error_best_effort_fails_drain_scope() { + let (tx, rx) = in_proc_channel(8); + let header = MppFrameHeader::task_metrics(1, 0, 1); + let sender = MppSender::new(Arc::new(tx)).clone_with_header(header); + let receiver = MppReceiver::new(Box::new(rx)); + let drain = DrainHandle::cooperative(0, vec![(ReceiverScope::Inbox, receiver)]); + + let mut task_rx = drain.take_execute_task_rx(1, 0).unwrap(); + + sender + .send_task_error_best_effort("worker failed: out of memory") + .await + .unwrap(); + + drain.try_drain_pass().unwrap(); + + let recv_item = task_rx.recv().await; + assert!(recv_item.is_some()); + let err = recv_item.unwrap().unwrap_err(); + assert!( + err.to_string().contains("worker failed: out of memory"), + "expected error string, got: {err}" + ); + } + impl DrainBuffer { /// Block until a batch is available, EOF is reached, or the buffer is cancelled. /// @@ -2535,13 +2661,14 @@ mod tests { | RecvBatchOutcome::TaskMetrics { .. } | RecvBatchOutcome::SetPlan { .. } | RecvBatchOutcome::ExecuteTask { .. } - | RecvBatchOutcome::Cancel { .. } => {} + | RecvBatchOutcome::Cancel { .. } + | RecvBatchOutcome::TaskError { .. } => {} RecvBatchOutcome::Empty => {} RecvBatchOutcome::Detached => { done[i] = true; buffer.notify_source_done(); } - RecvBatchOutcome::Error(e) => { + RecvBatchOutcome::TransportError(e) => { // Treat a decode error as a fatal detach for this source // so the consumer can observe EOF and abort the query. done[i] = true; @@ -3030,7 +3157,7 @@ mod tests { .try_recv() .expect("request frame must be delivered") .expect("delivery must not be a failure"); - let (decoded, _headers) = got.into_parts().expect("into_parts"); + let (decoded, _headers) = got.frame.into_parts().expect("into_parts"); assert_eq!(decoded.task_key.unwrap().task_number, 1); } @@ -3521,7 +3648,7 @@ mod tests { let receiver = MppReceiver::new(ReplayChannel::over(vec![orphan])); assert!(matches!( receiver.try_recv_batch(), - RecvBatchOutcome::Error(_) + RecvBatchOutcome::TransportError(_) )); let mut first = Vec::new(); @@ -3531,7 +3658,7 @@ mod tests { let receiver = MppReceiver::new(ReplayChannel::over(vec![first, skipped])); assert!(matches!( receiver.try_recv_batch(), - RecvBatchOutcome::Error(_) + RecvBatchOutcome::TransportError(_) )); } From 8aa6afe0082dfde6fe01a111526db794db6d44df Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Wed, 12 Aug 2026 13:28:41 +0900 Subject: [PATCH 15/15] Update to datafusion@7e015b74af0928187a77cdf9df23b587e349ebf9. --- Cargo.lock | 306 ++++++++++-------- Cargo.toml | 4 +- .../benches/broadcast_cache_scenarios.rs | 9 + cli/Cargo.toml | 2 +- examples/custom_execution_plan.rs | 11 + examples/custom_worker_url_routing.rs | 12 +- examples/work_unit_feed.rs | 9 + rust-toolchain.toml | 2 +- src/coordinator/distributed.rs | 9 + .../distributed_query_planner.rs | 19 +- .../normalize_collect_joins.rs | 4 +- src/execution_plans/broadcast.rs | 9 + .../children_isolator_union.rs | 11 + src/execution_plans/distributed_leaf.rs | 9 + src/execution_plans/metrics.rs | 9 + src/execution_plans/network_broadcast.rs | 12 + src/execution_plans/network_coalesce.rs | 9 + src/execution_plans/network_shuffle.rs | 12 + src/execution_plans/sampler.rs | 10 + src/explain_analyze.rs | 9 + src/test_utils/mock_exec.rs | 11 + src/test_utils/routing.rs | 11 + src/test_utils/test_work_unit_feed.rs | 9 + src/test_utils/work_unit_file_scan.rs | 18 +- tests/custom_config_extension.rs | 11 + tests/custom_extension_codec.rs | 11 + tests/error_propagation.rs | 11 + tests/multi_task_collect_join_repros.rs | 2 +- tests/stateful_execution_plan.rs | 11 + 29 files changed, 412 insertions(+), 160 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2df9879f..c86f624e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,7 +132,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -143,7 +143,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -193,9 +193,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", "arrow-buffer", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" +checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" dependencies = [ "arrow-array", "arrow-buffer", @@ -273,21 +273,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -296,7 +296,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -307,9 +307,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ "arrow-array", "arrow-cast", @@ -322,9 +322,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -345,7 +345,7 @@ dependencies = [ "arrow-data", "arrow-ipc", "arrow-schema", - "base64", + "base64 0.22.1", "bytes", "futures", "prost", @@ -356,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -372,9 +372,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -397,9 +397,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -423,9 +423,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "serde_core", "serde_json", @@ -433,9 +433,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", "arrow-array", @@ -447,9 +447,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -1010,6 +1010,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1028,7 +1034,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -1802,8 +1808,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-schema", @@ -1855,8 +1861,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -1879,8 +1885,8 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -1902,8 +1908,8 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -1929,8 +1935,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-ipc", @@ -1956,8 +1962,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "futures", "log", @@ -1966,8 +1972,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-compression", @@ -1983,6 +1989,7 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", @@ -2001,8 +2008,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-ipc", @@ -2015,6 +2022,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "itertools 0.15.0", @@ -2024,8 +2032,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-avro", @@ -2042,8 +2050,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -2055,6 +2063,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2064,8 +2073,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -2077,6 +2086,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -2086,8 +2096,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-schema", @@ -2104,6 +2114,7 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", @@ -2224,13 +2235,13 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" [[package]] name = "datafusion-execution" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-buffer", @@ -2255,8 +2266,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-schema", @@ -2268,6 +2279,8 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "indexmap", "itertools 0.15.0", "recursive", @@ -2277,8 +2290,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2288,12 +2301,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -2319,8 +2332,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2332,14 +2345,15 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2349,8 +2363,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-ord", @@ -2373,8 +2387,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "async-trait", @@ -2388,8 +2402,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2404,8 +2418,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2413,18 +2427,18 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "chrono", @@ -2442,8 +2456,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2464,8 +2478,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2478,8 +2492,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "chrono", @@ -2495,8 +2509,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2507,14 +2521,15 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", + "datafusion-session", "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "arrow-data", @@ -2549,11 +2564,10 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", - "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -2576,8 +2590,8 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2586,17 +2600,18 @@ dependencies = [ [[package]] name = "datafusion-proto-models" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ + "datafusion-common", "datafusion-proto-common", "prost", ] [[package]] name = "datafusion-pruning" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "datafusion-common", @@ -2610,9 +2625,10 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2623,8 +2639,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" -source = "git+https://github.com/apache/datafusion?rev=8d680db14a9134837bbd3d53497dd58c985c51a7#8d680db14a9134837bbd3d53497dd58c985c51a7" +version = "54.1.0" +source = "git+https://github.com/apache/datafusion?rev=7e015b74af0928187a77cdf9df23b587e349ebf9#7e015b74af0928187a77cdf9df23b587e349ebf9" dependencies = [ "arrow", "bigdecimal", @@ -2752,7 +2768,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2844,7 +2860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3464,7 +3480,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3698,7 +3714,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4011,9 +4027,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -4228,6 +4244,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4298,7 +4324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -4449,9 +4475,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.1.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash", "arrow-array", @@ -4460,7 +4486,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -4469,11 +4495,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "ring", "seq-macro", "simdutf8", @@ -4483,12 +4508,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pbkdf2" version = "0.12.2" @@ -4811,7 +4830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -5189,7 +5208,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -5275,7 +5294,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5612,7 +5631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5653,7 +5672,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5759,6 +5778,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5831,7 +5861,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5862,7 +5892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.11.1", "fancy-regex", "filedescriptor", @@ -6145,7 +6175,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.0", @@ -6704,7 +6734,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7a1deefaf..687933ecf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,8 @@ members = [ ] [workspace.dependencies] -datafusion = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } -datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } +datafusion = { git = "https://github.com/apache/datafusion", rev = "7e015b74af0928187a77cdf9df23b587e349ebf9", default-features = false } +datafusion-proto = { git = "https://github.com/apache/datafusion", rev = "7e015b74af0928187a77cdf9df23b587e349ebf9", default-features = false } [package] name = "datafusion-distributed" diff --git a/benchmarks/benches/broadcast_cache_scenarios.rs b/benchmarks/benches/broadcast_cache_scenarios.rs index 45a4fd48d..779257596 100644 --- a/benchmarks/benches/broadcast_cache_scenarios.rs +++ b/benchmarks/benches/broadcast_cache_scenarios.rs @@ -133,6 +133,15 @@ impl ExecutionPlan for SyntheticExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _children: Vec>, diff --git a/cli/Cargo.toml b/cli/Cargo.toml index fb2d4430d..2434b47c6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,7 +6,7 @@ edition = "2024" [dependencies] datafusion = { workspace = true } datafusion-distributed = { path = "..", features = ["avro", "integration"] } -datafusion-cli = { git = "https://github.com/apache/datafusion", rev = "8d680db14a9134837bbd3d53497dd58c985c51a7", default-features = false } +datafusion-cli = { git = "https://github.com/apache/datafusion", rev = "7e015b74af0928187a77cdf9df23b587e349ebf9", default-features = false } tokio = { version = "1.48", features = ["full"] } clap = { version = "4", features = ["derive"] } env_logger = "0.11" diff --git a/examples/custom_execution_plan.rs b/examples/custom_execution_plan.rs index 61ae45dfb..4b3e690fe 100644 --- a/examples/custom_execution_plan.rs +++ b/examples/custom_execution_plan.rs @@ -172,6 +172,17 @@ impl ExecutionPlan for NumbersExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _: Vec>, diff --git a/examples/custom_worker_url_routing.rs b/examples/custom_worker_url_routing.rs index 62f2e9740..7b9e5b2f6 100644 --- a/examples/custom_worker_url_routing.rs +++ b/examples/custom_worker_url_routing.rs @@ -45,7 +45,6 @@ use datafusion_distributed::{ DistributedLeafExec, RouteTasksEvent, RouteTasksEventResponse, ScaleUpLeafNodeEvent, ScaleUpLeafNodeEventResponse, SessionStateBuilderExt, WorkerQueryContext, display_plan_ascii, }; -use datafusion_proto::TryFromProto; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf; use futures::TryStreamExt; @@ -94,6 +93,15 @@ impl ExecutionPlan for CacheExec { vec![&self.child] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, mut children: Vec>, @@ -152,7 +160,7 @@ impl ExecutionPlan for CacheExec { fn hash_key(file_group: &FileGroup) -> usize { let mut hasher = DefaultHasher::new(); for file in file_group.files() { - let serialized = protobuf::PartitionedFile::try_from_proto(file).unwrap(); + let serialized = protobuf::PartitionedFile::try_from(file).unwrap(); hasher.write(&serialized.encode_to_vec()); } hasher.finish() as usize diff --git a/examples/work_unit_feed.rs b/examples/work_unit_feed.rs index 3daa12223..16096aae5 100644 --- a/examples/work_unit_feed.rs +++ b/examples/work_unit_feed.rs @@ -148,6 +148,15 @@ impl ExecutionPlan for RemoteScanExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _: Vec>, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 28130b1dc..cc6d07f37 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.91" +channel = "1.94.0" profile = "default" diff --git a/src/coordinator/distributed.rs b/src/coordinator/distributed.rs index a2cfa2ebd..84a957b96 100644 --- a/src/coordinator/distributed.rs +++ b/src/coordinator/distributed.rs @@ -162,6 +162,15 @@ impl ExecutionPlan for DistributedExec { vec![&self.base_plan] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/distributed_planner/distributed_query_planner.rs b/src/distributed_planner/distributed_query_planner.rs index faf5f447c..14996e9ea 100644 --- a/src/distributed_planner/distributed_query_planner.rs +++ b/src/distributed_planner/distributed_query_planner.rs @@ -54,25 +54,28 @@ pub(crate) struct DistributedQueryPlanner { pub(crate) prev: Option>, } +use datafusion::catalog::Session; + #[async_trait] impl QueryPlanner for DistributedQueryPlanner { async fn create_physical_plan( &self, logical_plan: &LogicalPlan, - session_state: &SessionState, + session: &dyn Session, ) -> datafusion::common::Result> { + let session_state = session + .as_any() + .downcast_ref::() + .ok_or_else(|| { + datafusion::common::DataFusionError::Internal("Expected SessionState".to_string()) + })?; let original_plan = match &self.prev { None => { // Use the default physical planner. let planner = DefaultPhysicalPlanner::default(); - planner - .create_physical_plan(logical_plan, session_state) - .await? - } - Some(prev) => { - prev.create_physical_plan(logical_plan, session_state) - .await? + planner.create_physical_plan(logical_plan, session).await? } + Some(prev) => prev.create_physical_plan(logical_plan, session).await?, }; create_distributed_plan(original_plan, session_state).await diff --git a/src/distributed_planner/normalize_collect_joins.rs b/src/distributed_planner/normalize_collect_joins.rs index fd4398e6a..f8fd50563 100644 --- a/src/distributed_planner/normalize_collect_joins.rs +++ b/src/distributed_planner/normalize_collect_joins.rs @@ -347,8 +347,8 @@ mod tests { // this holds only if the converted join kept the original filter (the same instance // the probe-side scan subscribes to), not a lookalike replacement. assert_eq!( - original.dynamic_filter_expr().unwrap(), - converted.dynamic_filter_expr().unwrap() + original.dynamic_expressions_produced(), + converted.dynamic_expressions_produced() ); } diff --git a/src/execution_plans/broadcast.rs b/src/execution_plans/broadcast.rs index 094b0e4a4..b4522f094 100644 --- a/src/execution_plans/broadcast.rs +++ b/src/execution_plans/broadcast.rs @@ -136,6 +136,15 @@ impl ExecutionPlan for BroadcastExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/children_isolator_union.rs b/src/execution_plans/children_isolator_union.rs index 3b0fc4c9e..886a0d484 100644 --- a/src/execution_plans/children_isolator_union.rs +++ b/src/execution_plans/children_isolator_union.rs @@ -275,6 +275,17 @@ impl ExecutionPlan for ChildrenIsolatorUnionExec { &self.properties } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/distributed_leaf.rs b/src/execution_plans/distributed_leaf.rs index 4c3bea511..faaefe1d8 100644 --- a/src/execution_plans/distributed_leaf.rs +++ b/src/execution_plans/distributed_leaf.rs @@ -158,6 +158,15 @@ impl ExecutionPlan for DistributedLeafExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/metrics.rs b/src/execution_plans/metrics.rs index de97ed049..54de2a392 100644 --- a/src/execution_plans/metrics.rs +++ b/src/execution_plans/metrics.rs @@ -53,6 +53,15 @@ impl ExecutionPlan for MetricsWrapperExec { self.inner.children() } + fn apply_expressions( + &self, + f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + self.inner.apply_expressions(f) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/network_broadcast.rs b/src/execution_plans/network_broadcast.rs index aa84ea788..9ae3b1a67 100644 --- a/src/execution_plans/network_broadcast.rs +++ b/src/execution_plans/network_broadcast.rs @@ -214,6 +214,18 @@ impl ExecutionPlan for NetworkBroadcastExec { } } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result< + datafusion::common::tree_node::TreeNodeRecursion, + DataFusionError, + >, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 4379910a5..ab2ac64d5 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -222,6 +222,15 @@ impl ExecutionPlan for NetworkCoalesceExec { } } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index c7a645aef..83518caea 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -191,6 +191,18 @@ impl ExecutionPlan for NetworkShuffleExec { } } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result< + datafusion::common::tree_node::TreeNodeRecursion, + DataFusionError, + >, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/execution_plans/sampler.rs b/src/execution_plans/sampler.rs index 8eb5c3642..a79cb9dde 100644 --- a/src/execution_plans/sampler.rs +++ b/src/execution_plans/sampler.rs @@ -255,6 +255,7 @@ impl PartitionSampler { // The input produced nothing at all: the stream is exhausted, so this partition // reached EOS with zero rows. reporter.load_info.reached_eos = true; + reporter.report(&peek); return Ok(peek.chain(input_stream).boxed()); }; let _guard = elapsed_compute.timer(); @@ -535,6 +536,15 @@ impl ExecutionPlan for SamplerExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/explain_analyze.rs b/src/explain_analyze.rs index 7e321244b..72f8cf83a 100644 --- a/src/explain_analyze.rs +++ b/src/explain_analyze.rs @@ -56,6 +56,15 @@ impl ExecutionPlan for DistributedAnalyzeExec { vec![&self.input] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn required_input_distribution(&self) -> Vec { vec![Distribution::UnspecifiedDistribution] } diff --git a/src/test_utils/mock_exec.rs b/src/test_utils/mock_exec.rs index fa990e8f1..98f305335 100644 --- a/src/test_utils/mock_exec.rs +++ b/src/test_utils/mock_exec.rs @@ -151,6 +151,17 @@ impl ExecutionPlan for MockExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _: Vec>, diff --git a/src/test_utils/routing.rs b/src/test_utils/routing.rs index 9f515e57a..562ed1468 100644 --- a/src/test_utils/routing.rs +++ b/src/test_utils/routing.rs @@ -182,6 +182,17 @@ impl ExecutionPlan for URLEmitterExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::error::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::error::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _: Vec>, diff --git a/src/test_utils/test_work_unit_feed.rs b/src/test_utils/test_work_unit_feed.rs index 7f4a5e1f0..35a9bb44b 100644 --- a/src/test_utils/test_work_unit_feed.rs +++ b/src/test_utils/test_work_unit_feed.rs @@ -389,6 +389,15 @@ impl ExecutionPlan for RowGeneratorExec { vec![] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, _: Vec>, diff --git a/src/test_utils/work_unit_file_scan.rs b/src/test_utils/work_unit_file_scan.rs index 6f6c36dd3..4fb784406 100644 --- a/src/test_utils/work_unit_file_scan.rs +++ b/src/test_utils/work_unit_file_scan.rs @@ -29,8 +29,8 @@ use datafusion_proto::physical_plan::{ DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalPlanNodeExt, PhysicalProtoConverterExtension, }; +use datafusion_proto::protobuf as df_proto; use datafusion_proto::protobuf::proto_error; -use datafusion_proto::{TryFromProto, protobuf as df_proto}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use prost::Message; @@ -78,7 +78,7 @@ impl WorkUnitFeedProvider for FileScanWorkUnitProvider { return Ok(futures::stream::empty().boxed()); }; let stream = futures::stream::iter(file_group.files().to_vec()).map(|file| { - let file_proto = df_proto::PartitionedFile::try_from_proto(&file) + let file_proto = df_proto::PartitionedFile::try_from(&file) .map_err(|e| internal_datafusion_err!("{e}"))?; Ok(FileScanWorkUnit { file: Some(file_proto), @@ -132,9 +132,8 @@ impl DataSource for WorkUnitFileScanConfig { .map(move |work_unit| { let file = work_unit?.file.expect("missing file"); - let df_file = - datafusion::datasource::listing::PartitionedFile::try_from_proto(&file) - .map_err(|e: DataFusionError| e)?; + let df_file = datafusion::datasource::listing::PartitionedFile::try_from(&file) + .map_err(|e: DataFusionError| e)?; let single_file_group = FileGroup::from(vec![df_file]); let new_config = FileScanConfigBuilder::from(inner.clone()) @@ -159,6 +158,15 @@ impl DataSource for WorkUnitFileScanConfig { } } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> Result, + ) -> Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn repartitioned( &self, _target_partitions: usize, diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index 50ccb0fe9..fe7209da8 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -161,6 +161,17 @@ mod tests { vec![&self.child] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index 7c23c8d8b..487a56fcf 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -110,6 +110,17 @@ mod tests { vec![&self.child] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index 052895940..534c3abf0 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -110,6 +110,17 @@ mod tests { vec![&self.child] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/tests/multi_task_collect_join_repros.rs b/tests/multi_task_collect_join_repros.rs index 23580e746..311ef5b0f 100644 --- a/tests/multi_task_collect_join_repros.rs +++ b/tests/multi_task_collect_join_repros.rs @@ -103,7 +103,7 @@ mod tests { HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware CoalescePartitionsExec DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/build_side/part-0.parquet, /target/multi_task_collect_join_repros/build_side/part-1.parquet], [/target/multi_task_collect_join_repros/build_side/part-2.parquet, /target/multi_task_collect_join_repros/build_side/part-3.parquet]]}, projection=[id], file_type=parquet - DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet, /target/multi_task_collect_join_repros/probe_side/part-1.parquet], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet, /target/multi_task_collect_join_repros/probe_side/part-3.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ id@0 IS NULL OR id@0 >= 0 AND id@0 <= 99 AND id@0 IN (SET) ([]) ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@0 > 0 OR id_null_count@0 != row_count@2 AND id_max@1 >= 0 AND id_null_count@0 != row_count@2 AND id_min@3 <= 99, required_guarantees=[] + DataSourceExec: file_groups={2 groups: [[/target/multi_task_collect_join_repros/probe_side/part-0.parquet, /target/multi_task_collect_join_repros/probe_side/part-1.parquet], [/target/multi_task_collect_join_repros/probe_side/part-2.parquet, /target/multi_task_collect_join_repros/probe_side/part-3.parquet]]}, projection=[id], file_type=parquet ") } diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index 926053815..f2069d520 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -137,6 +137,17 @@ mod tests { vec![&self.child] } + fn apply_expressions( + &self, + _f: &mut dyn FnMut( + &Arc, + ) -> datafusion::common::Result< + datafusion::common::tree_node::TreeNodeRecursion, + >, + ) -> datafusion::common::Result { + Ok(datafusion::common::tree_node::TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>,