From c66ee3b7dff589a353ad58ca474f2f4b391f243c Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Fri, 11 Sep 2026 15:09:44 +0200 Subject: [PATCH 01/18] tests: stage all Altinity#1844 alias-marker tests for 26.6 triage Tests only, no source changes. Purpose is to establish which of the `__aliasMarker` regressions from https://github.com/Altinity/ClickHouse/pull/1844 still reproduce on `antalya-26.6`, which already carries the upstream fixes that landed after that PR was written: * https://github.com/ClickHouse/ClickHouse/pull/94644 (ALIAS column order) * https://github.com/ClickHouse/ClickHouse/pull/107675 (distributed ALIAS column order) * https://github.com/ClickHouse/ClickHouse/pull/107913 (`buildShardCollapseFanOut`) All 22 distinct test files from #1844 are staged: the 18 active ones plus the 4 unique files kept under `_local_files_and_notes/dropped_tests` there. The six dropped-test files that duplicate active ones byte-for-byte are staged once. Slot numbers from the 26.3 branch that are already taken on 26.6 are reassigned to the 05053-05063 range: 03843 -> 05053 03923 -> 05056 04286 -> 05059 03926 -> 05062 03920 -> 05054 03928 -> 05057 03924 -> 05060 03927 -> 05063 03921 -> 05055 03930 -> 05058 03925 -> 05061 `03648_alias_marker_with_mergeable_state.sh` is updated to the #1844 revision. It is a contract test for the reworked marker, not a bug reproducer, so it is expected to fail until the source change lands. Related: https://github.com/Altinity/ClickHouse/pull/1844 Related: https://github.com/ClickHouse/ClickHouse/pull/106402 --- ...lias_marker_with_mergeable_state.reference | 2 + ...03648_alias_marker_with_mergeable_state.sh | 21 +- .../03842_hybrid_alias_issue_1424.reference | 42 +++ .../03842_hybrid_alias_issue_1424.sql | 202 +++++++++++ ..._distributed_nested_alias_marker.reference | 4 + .../03844_distributed_nested_alias_marker.sql | 34 ++ ...buted_global_in_join_alias_chain.reference | 8 + ...distributed_global_in_join_alias_chain.sql | 34 ++ ...global_in_alias_marker_collision.reference | 2 + ...buted_global_in_alias_marker_collision.sql | 56 +++ ...replicas_second_hop_alias_marker.reference | 4 + ...allel_replicas_second_hop_alias_marker.sql | 51 +++ ...931_parallel_replicas_alias_swap.reference | 9 + .../03931_parallel_replicas_alias_swap.sql | 37 ++ ...32_distributed_alias_strict_name.reference | 9 + .../03932_distributed_alias_strict_name.sql | 27 ++ ...rker_direct_use_no_logical_error.reference | 14 + ...ias_marker_direct_use_no_logical_error.sql | 31 ++ ...uted_alias_marker_setting_effect.reference | 15 + ...istributed_alias_marker_setting_effect.sql | 39 +++ ...age_merge_over_distributed_alias.reference | 20 ++ ...1_storage_merge_over_distributed_alias.sql | 58 +++ ...istributed_alias_same_expression.reference | 15 + ...5053_distributed_alias_same_expression.sql | 67 ++++ ...buted_global_alias_marker_matrix.reference | 35 ++ ...distributed_global_alias_marker_matrix.sql | 297 ++++++++++++++++ ..._over_distributed_double_aliases.reference | 56 +++ ...ibuted_over_distributed_double_aliases.sql | 92 +++++ ...nown_table_issues_1208_1209_1422.reference | 8 + ...id_unknown_table_issues_1208_1209_1422.sql | 118 +++++++ ...ributed_alias_marker_column_swap.reference | 20 ++ ...r_distributed_alias_marker_column_swap.sql | 58 +++ ...8_distributed_alias_swap_planner.reference | 15 + .../05058_distributed_alias_swap_planner.sql | 34 ++ ...ted_alias_merge_over_distributed.reference | 9 + ...59_dotted_alias_merge_over_distributed.sql | 64 ++++ ...ybrid_unknown_table_exact_schema.reference | 30 ++ ...5060_hybrid_unknown_table_exact_schema.sql | 329 ++++++++++++++++++ ...alias_column_swap_without_marker.reference | 56 +++ ...buted_alias_column_swap_without_marker.sql | 96 +++++ ...l_replicas_dod_alias_column_swap.reference | 20 ++ ...arallel_replicas_dod_alias_column_swap.sql | 94 +++++ ...lias_marker_explicit_column_swap.reference | 32 ++ ...uted_alias_marker_explicit_column_swap.sql | 82 +++++ 44 files changed, 2340 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference create mode 100644 tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql create mode 100644 tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference create mode 100644 tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql create mode 100644 tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference create mode 100644 tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql create mode 100644 tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference create mode 100644 tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql create mode 100644 tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference create mode 100644 tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql create mode 100644 tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference create mode 100644 tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql create mode 100644 tests/queries/0_stateless/03932_distributed_alias_strict_name.reference create mode 100644 tests/queries/0_stateless/03932_distributed_alias_strict_name.sql create mode 100644 tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference create mode 100644 tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql create mode 100644 tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference create mode 100644 tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql create mode 100644 tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference create mode 100644 tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql create mode 100644 tests/queries/0_stateless/05053_distributed_alias_same_expression.reference create mode 100644 tests/queries/0_stateless/05053_distributed_alias_same_expression.sql create mode 100644 tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference create mode 100644 tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql create mode 100644 tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference create mode 100644 tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql create mode 100644 tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference create mode 100644 tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql create mode 100644 tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference create mode 100644 tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql create mode 100644 tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference create mode 100644 tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql create mode 100644 tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference create mode 100644 tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql create mode 100644 tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference create mode 100644 tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql create mode 100644 tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference create mode 100644 tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql create mode 100644 tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference create mode 100644 tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql create mode 100644 tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference create mode 100644 tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference index 58bf6a7ec74b..5f061a829b23 100644 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference @@ -2,6 +2,8 @@ Header: sum(foo) AggregateFunction(sum, Int64) ---- stage: with_mergeable_state (analyzer=0) ---- Expected error: Function __aliasMarker is internal and supported only with the analyzer +---- explicit __aliasMarker in user query (analyzer=1) ---- +Explicit __aliasMarker call is allowed ---- stage: complete (analyzer=1) ---- Header: x Int64 ---- stage: fetch_columns (analyzer=1) ---- diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh index 66974be38517..fb0580e796f0 100755 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh @@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh echo "---- stage: with_mergeable_state (analyzer=1, setting=enable_alias_marker=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state --multiquery 2>&1 <<'EOF' | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state --multiquery 2>&1 <<'EOF' | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' SET enable_alias_marker=1; EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x @@ -22,27 +22,36 @@ else echo "${alias_marker_error_output}" fi +echo "---- explicit __aliasMarker in user query (analyzer=1) ----" +if $CLICKHOUSE_CLIENT --enable_analyzer=1 --query \ + "SELECT __aliasMarker(number*2-3,'foo') FROM numbers(1)" >/dev/null 2>&1 +then + echo "Explicit __aliasMarker call is allowed" +else + echo "Unexpected error for explicit __aliasMarker call" +fi + echo "---- stage: complete (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage complete --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage complete --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: fetch_columns (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage fetch_columns --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage fetch_columns --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state_after_aggregation (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state_after_aggregation --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state_after_aggregation --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state_after_aggregation_and_limit (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state_after_aggregation_and_limit --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state_after_aggregation_and_limit --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10) GROUP BY intDiv(number,10) AS y ORDER BY y LIMIT 10" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference new file mode 100644 index 000000000000..6f78da4c4f59 --- /dev/null +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference @@ -0,0 +1,42 @@ +max in subquery +4294967294 +sum in subquery +-4921211434 +cte min with predicate +679772422 +cte with limit +-2147483648 -4294967296 +-1762862292 -574613778 +-1329695183 -1573638336 +-221724287 679772422 +0 0 +550067609 -3048000734 +1084637461 3417479706 +1169291374 -3082049462 +1899628504 -740161250 +2147483647 4294967294 +cte without limit +-2147483648 -4294967296 +-1762862292 -574613778 +-1329695183 -1573638336 +-221724287 679772422 +0 0 +550067609 -3048000734 +1084637461 3417479706 +1169291374 -3082049462 +1899628504 -740161250 +2147483647 4294967294 +group by in subquery +10 10 +intersect with order by +-221724287 679772422 +1084637461 3417479706 +2147483647 4294967294 +intersect without order by +-221724287 679772422 +1084637461 3417479706 +2147483647 4294967294 +constant alias in subquery +9 7 32 +constant alias predicate +2 diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql new file mode 100644 index 000000000000..8b9cf9182896 --- /dev/null +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql @@ -0,0 +1,202 @@ +SET allow_experimental_hybrid_table = 1, enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE IF EXISTS test_hybrid_issue_1424; +DROP TABLE IF EXISTS test_hybrid_issue_1424_left; +DROP TABLE IF EXISTS test_hybrid_issue_1424_right; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const_left; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const_right; + +CREATE TABLE test_hybrid_issue_1424_left +( + id Int32, + value Int32, + date_col Date, + computed ALIAS value * 2 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_left VALUES + (toInt32(2147483647), toInt32(2147483647), toDate('2149-06-06')), + (toInt32(-2147483648), toInt32(-2147483648), toDate('1970-01-01')), + (toInt32(0), toInt32(0), '1970-01-01'), + (toInt32(1084637461), toInt32(1708739853), toDate(1335613783)), + (toInt32(-221724287), toInt32(339886211), toDate(1294089763)), + (toInt32(-1762862292), toInt32(-287306889), toDate(1375707465)), + (toInt32(1169291374), toInt32(-1541024731), toDate(1082126480)), + (toInt32(-1329695183), toInt32(-786819168), toDate(1226000164)), + (toInt32(1899628504), toInt32(-370080625), toDate(1179050966)), + (toInt32(550067609), toInt32(-1524000367), toDate(1410654931)); + +CREATE TABLE test_hybrid_issue_1424_right +( + id Int32, + value Int32, + date_col Date, + computed ALIAS value * 2 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_right VALUES + (toInt32(2147483647), toInt32(2147483647), toDate('2149-06-06')), + (toInt32(-2147483648), toInt32(-2147483648), toDate('1970-01-01')), + (toInt32(0), toInt32(0), '1970-01-01'), + (toInt32(1084637461), toInt32(1708739853), toDate(1335613783)), + (toInt32(-221724287), toInt32(339886211), toDate(1294089763)), + (toInt32(-1762862292), toInt32(-287306889), toDate(1375707465)), + (toInt32(1169291374), toInt32(-1541024731), toDate(1082126480)), + (toInt32(-1329695183), toInt32(-786819168), toDate(1226000164)), + (toInt32(1899628504), toInt32(-370080625), toDate(1179050966)), + (toInt32(550067609), toInt32(-1524000367), toDate(1410654931)); + +CREATE TABLE test_hybrid_issue_1424 +( + id Int32, + value Int32, + date_col Date, + computed Int64 +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_left'), date_col >= '2025-01-15', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_right'), date_col < '2025-01-15' +); + +SELECT 'max in subquery'; +SELECT max_computed FROM (SELECT max(computed) AS max_computed FROM test_hybrid_issue_1424); + +SELECT 'sum in subquery'; +SELECT sum_computed FROM (SELECT sum(computed) AS sum_computed FROM test_hybrid_issue_1424); + +SELECT 'cte min with predicate'; +WITH cte AS +( + SELECT min(computed) AS min_computed + FROM test_hybrid_issue_1424 + WHERE computed > 50 +) +SELECT * FROM cte; + +SELECT 'cte with limit'; +WITH ranked AS +( + SELECT id, computed + FROM test_hybrid_issue_1424 + LIMIT 10 +) +SELECT * +FROM ranked +ORDER BY id ASC; + +SELECT 'cte without limit'; +WITH ranked AS +( + SELECT id, computed + FROM test_hybrid_issue_1424 +) +SELECT * +FROM ranked +ORDER BY id ASC; + +SELECT 'group by in subquery'; +WITH monthly AS +( + SELECT count() AS cnt + FROM test_hybrid_issue_1424 + GROUP BY computed +) +SELECT sum(cnt), count() FROM monthly; + +SELECT 'intersect with order by'; +SELECT * +FROM +( + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE computed > 100 + INTERSECT + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE value > 50 +) +ORDER BY id; + +SELECT 'intersect without order by'; +SELECT * +FROM +( + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE computed > 100 + INTERSECT + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE value > 50 +) +ORDER BY id; + +CREATE TABLE test_hybrid_issue_1424_const_left +( + id Int32, + value Int32, + date_col Date, + computed ALIAS toInt64(7) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_const_left VALUES + (1, 1, toDate('2025-01-15')), + (2, 2, toDate('2025-02-01')); + +CREATE TABLE test_hybrid_issue_1424_const_right +( + id Int32, + value Int32, + date_col Date, + computed ALIAS toInt64(9) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_const_right VALUES + (3, 3, toDate('2024-12-31')), + (4, 4, toDate('2020-01-01')); + +CREATE TABLE test_hybrid_issue_1424_const +( + id Int32, + value Int32, + date_col Date, + computed Int64 +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_const_left'), date_col >= '2025-01-15', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_const_right'), date_col < '2025-01-15' +); + +SELECT 'constant alias in subquery'; +SELECT max_computed, min_computed, sum_computed +FROM +( + SELECT + max(computed) AS max_computed, + min(computed) AS min_computed, + sum(computed) AS sum_computed + FROM test_hybrid_issue_1424_const +); + +SELECT 'constant alias predicate'; +SELECT count() FROM test_hybrid_issue_1424_const WHERE computed = 9; + +DROP TABLE test_hybrid_issue_1424; +DROP TABLE test_hybrid_issue_1424_left; +DROP TABLE test_hybrid_issue_1424_right; +DROP TABLE test_hybrid_issue_1424_const; +DROP TABLE test_hybrid_issue_1424_const_left; +DROP TABLE test_hybrid_issue_1424_const_right; diff --git a/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference new file mode 100644 index 000000000000..7b05cb1e81a0 --- /dev/null +++ b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference @@ -0,0 +1,4 @@ +analyzer +x x +legacy +x x diff --git a/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql new file mode 100644 index 000000000000..b725acf38949 --- /dev/null +++ b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test_nested_alias_dist; +DROP TABLE IF EXISTS test_nested_alias_local; + +CREATE TABLE test_nested_alias_local +( + dt DateTime64(3), + base String, + a String ALIAS base, + b String ALIAS a +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_nested_alias_local VALUES ('1999-03-29T01:15:33', 'x'); + +CREATE TABLE test_nested_alias_dist AS test_nested_alias_local +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_nested_alias_local, rand()); + +SELECT 'analyzer'; +SELECT a, b +FROM test_nested_alias_dist +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1; + +SELECT 'legacy'; +SELECT a, b +FROM test_nested_alias_dist +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 0; + +DROP TABLE test_nested_alias_dist; +DROP TABLE test_nested_alias_local; diff --git a/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference new file mode 100644 index 000000000000..325078d71cc1 --- /dev/null +++ b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference @@ -0,0 +1,8 @@ +rewrite_in +1 +1 +rewrite_join +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql new file mode 100644 index 000000000000..9bd95d72fd20 --- /dev/null +++ b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test_global_alias_chain_dist; +DROP TABLE IF EXISTS test_global_alias_chain_local; + +CREATE TABLE test_global_alias_chain_local +( + id UInt64, + base UInt64, + a UInt64 ALIAS base, + b UInt64 ALIAS a +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_global_alias_chain_local VALUES (1, 1); + +CREATE TABLE test_global_alias_chain_dist AS test_global_alias_chain_local +ENGINE = Distributed('test_cluster_two_shards', currentDatabase(), test_global_alias_chain_local, rand()); + +SELECT 'rewrite_in'; +SELECT id +FROM test_global_alias_chain_dist +WHERE id IN (SELECT b FROM test_global_alias_chain_dist) +ORDER BY id +SETTINGS enable_analyzer = 1, distributed_product_mode = 'global'; + +SELECT 'rewrite_join'; +SELECT l.id +FROM test_global_alias_chain_dist AS l +INNER JOIN (SELECT b FROM test_global_alias_chain_dist) AS r ON l.id = r.b +ORDER BY l.id +SETTINGS enable_analyzer = 1, distributed_product_mode = 'global'; + +DROP TABLE test_global_alias_chain_dist; +DROP TABLE test_global_alias_chain_local; diff --git a/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference new file mode 100644 index 000000000000..9a3a29a69ce8 --- /dev/null +++ b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference @@ -0,0 +1,2 @@ +global_in_collision_check +1 diff --git a/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql new file mode 100644 index 000000000000..d47e6a304ba1 --- /dev/null +++ b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql @@ -0,0 +1,56 @@ +DROP TABLE IF EXISTS test_marker_collision_dist; +DROP TABLE IF EXISTS test_marker_collision_main; +DROP TABLE IF EXISTS test_marker_collision_left; +DROP TABLE IF EXISTS test_marker_collision_right; + +CREATE TABLE test_marker_collision_main +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_collision_main VALUES (1); + +CREATE TABLE test_marker_collision_left +( + id UInt64, + x UInt64, + b UInt64 ALIAS x +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_marker_collision_right +( + id UInt64, + y UInt64, + b UInt64 ALIAS y +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_collision_left VALUES (1, 1); +INSERT INTO test_marker_collision_right VALUES (1, 20); + +CREATE TABLE test_marker_collision_dist AS test_marker_collision_main +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_collision_main, rand()); + +SELECT 'global_in_collision_check'; +SELECT id +FROM test_marker_collision_dist +WHERE id GLOBAL IN +( + SELECT test_marker_collision_left.id + FROM test_marker_collision_left + INNER JOIN test_marker_collision_right + ON test_marker_collision_left.id = test_marker_collision_right.id + WHERE test_marker_collision_left.b + test_marker_collision_right.b = 21 +) +ORDER BY id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE test_marker_collision_dist; +DROP TABLE test_marker_collision_main; +DROP TABLE test_marker_collision_left; +DROP TABLE test_marker_collision_right; diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference new file mode 100644 index 000000000000..fbdae0d35623 --- /dev/null +++ b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference @@ -0,0 +1,4 @@ +single_replica_second_hop +1999-03-29 01:15:33.000 x x +parallel_replicas_second_hop +1999-03-29 01:15:33.000 x x diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql new file mode 100644 index 000000000000..5500c4904f82 --- /dev/null +++ b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql @@ -0,0 +1,51 @@ +-- Regression coverage for materialized __aliasMarker metadata across +-- remote -> Distributed -> parallel replicas fanout. + +DROP TABLE IF EXISTS test_alias_pr_second_hop_dist; +DROP TABLE IF EXISTS test_alias_pr_second_hop_local; + +CREATE TABLE test_alias_pr_second_hop_local +( + dt DateTime64(3), + base String, + alias_base_0 String ALIAS base, + alias_base_1 String ALIAS base +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_alias_pr_second_hop_local VALUES + ('1999-03-29T01:15:33', 'x'), + ('1999-03-29T01:15:34', 'y'); + +CREATE TABLE test_alias_pr_second_hop_dist AS test_alias_pr_second_hop_local +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_alias_pr_second_hop_local); + +SELECT 'single_replica_second_hop'; +SELECT dt, alias_base_0, alias_base_1 +FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) +ORDER BY dt +LIMIT 1 +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + enable_parallel_replicas = 1, + max_parallel_replicas = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1; + +SELECT 'parallel_replicas_second_hop'; +SELECT dt, alias_base_0, alias_base_1 +FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) +ORDER BY dt +LIMIT 1 +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + enable_parallel_replicas = 1, + max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1; + +DROP TABLE test_alias_pr_second_hop_dist; +DROP TABLE test_alias_pr_second_hop_local; diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference new file mode 100644 index 000000000000..14f9c770f714 --- /dev/null +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference @@ -0,0 +1,9 @@ +local +10 11 12 +20 21 22 +pr_ast +10 11 12 +20 21 22 +pr_plan +10 11 12 +20 21 22 diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql new file mode 100644 index 000000000000..f669631889c2 --- /dev/null +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql @@ -0,0 +1,37 @@ +-- Plain Distributed + parallel replicas (no Hybrid). Exercises the findParallelReplicasQuery +-- header reconciliation path with nested ALIAS columns. Correct result equals the single-node +-- ('local') result for both AST and serialized-plan transport. +-- +-- Determinism note: parallel replicas over a small non-replicated table can read the same rows on +-- several replicas under some (randomized) settings, duplicating output. GROUP BY x, a1, a2 +-- deduplicates that and keeps x in the required columns for the ALIAS expansion; ORDER BY x over +-- distinct values gives a total order. The test still fails if a1/a2 are swapped or wrong. +DROP TABLE IF EXISTS t_local_03931; +DROP TABLE IF EXISTS t_dist_03931; + +CREATE TABLE t_local_03931 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03931 VALUES (10), (20); + +CREATE TABLE t_dist_03931 AS t_local_03931 +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_local_03931); + +SELECT 'local'; +SELECT x, a1, a2 FROM t_local_03931 GROUP BY x, a1, a2 ORDER BY x; + +SELECT 'pr_ast'; +SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, + allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 0; + +SELECT 'pr_plan'; +SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, + allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 1; + +DROP TABLE t_dist_03931; +DROP TABLE t_local_03931; diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference b/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference new file mode 100644 index 000000000000..cddf594d4e31 --- /dev/null +++ b/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference @@ -0,0 +1,9 @@ +local +12 11 23 +22 21 43 +dist +12 11 23 +22 21 43 +dist_plan +12 11 23 +22 21 43 diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql b/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql new file mode 100644 index 000000000000..c094d28f01e0 --- /dev/null +++ b/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql @@ -0,0 +1,27 @@ +-- Plain Distributed (no Hybrid). Reorders alias columns and mixes a computed expression over +-- them. With strict name-based header reconciliation (positional fallback disabled), the result +-- must equal the single-node ('local') result for both AST and serialized-plan transport, and no +-- LOGICAL_ERROR must be raised. +DROP TABLE IF EXISTS t_local_03932; +DROP TABLE IF EXISTS t_dist_03932; + +CREATE TABLE t_local_03932 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03932 VALUES (10), (20); + +CREATE TABLE t_dist_03932 AS t_local_03932 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03932); + +SELECT 'local'; +SELECT a2, a1, a1 + a2 AS s FROM t_local_03932 ORDER BY x; + +SELECT 'dist'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'dist_plan'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_03932; +DROP TABLE t_local_03932; diff --git a/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference new file mode 100644 index 000000000000..f3f736b7dea2 --- /dev/null +++ b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference @@ -0,0 +1,14 @@ +2arg_identity +42 +lambda_local +[1] +[2] +[3] +lambda_over_distributed +[1] +[2] +[3] +lambda_over_distributed_plan +[1] +[2] +[3] diff --git a/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql new file mode 100644 index 000000000000..e327c442397d --- /dev/null +++ b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql @@ -0,0 +1,31 @@ +-- __aliasMarker is an internal pass-through identity function. Direct use from SQL must not +-- raise a server-side LOGICAL_ERROR (which would abort under abort_on_logical_error / sanitizers), +-- in particular inside a lambda over a Distributed table where the marker's column argument +-- resolves to a lambda parameter with no table source. +DROP TABLE IF EXISTS t_local_03933; +DROP TABLE IF EXISTS t_dist_03933; + +CREATE TABLE t_local_03933 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03933 VALUES (1), (2), (3); + +CREATE TABLE t_dist_03933 AS t_local_03933 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03933); + +SELECT '2arg_identity'; +SELECT __aliasMarker(42, 'anything'); + +SELECT 'lambda_local'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr FROM t_local_03933 ORDER BY x; + +SELECT 'lambda_over_distributed'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr +FROM t_dist_03933 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda_over_distributed_plan'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr +FROM t_dist_03933 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_03933; +DROP TABLE t_local_03933; diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference new file mode 100644 index 000000000000..cc0e49a8fdf9 --- /dev/null +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference @@ -0,0 +1,15 @@ +marker_on +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +marker_off_reintroduces_swap diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql new file mode 100644 index 000000000000..4100828c9eb5 --- /dev/null +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql @@ -0,0 +1,39 @@ +-- Demonstrates that enable_alias_marker is a correctness toggle for distributed ALIAS columns. +-- Distributed-over-distributed with a String ALIAS (`a_str`) and a UInt64 ALIAS (`inner_c`): +-- * marker ON -> columns reconciled by name, correct results. +-- * marker OFF -> the inlined ALIAS expansion swaps columns; the String 'aaaa' is routed into +-- the UInt64 `inner_c` slot and the query fails with CANNOT_PARSE_TEXT. +DROP TABLE IF EXISTS t_se_local; +DROP TABLE IF EXISTS t_se_inner; +DROP TABLE IF EXISTS t_se_outer; + +CREATE TABLE t_se_local (x UInt64) ENGINE = MergeTree() ORDER BY x; +INSERT INTO t_se_local VALUES (1), (2), (10); + +CREATE TABLE t_se_inner (x UInt64, inner_c UInt64 ALIAS x + 1) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_local); + +CREATE TABLE t_se_outer (x UInt64, inner_c UInt64, a_str String ALIAS 'aaaa') +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_inner); + +-- serialize_query_plan is pinned to 0 throughout: this test targets the AST-path alias marker. +-- On the serialized-plan path the header is reconciled by name regardless of the marker, so the +-- marker_off swap below does not occur there; the "distributed plan" CI flavor would otherwise +-- force the plan path on and the marker_off query would succeed instead of erroring. +SELECT 'marker_on'; +SELECT x, a_str, inner_c +FROM t_se_outer +ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0 +FORMAT TSVWithNames; + +SELECT 'marker_off_reintroduces_swap'; +-- No output format header here: the query errors mid-execution, so it must not stream a header. +SELECT x, a_str, inner_c +FROM t_se_outer +ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 0, prefer_localhost_replica = 0, serialize_query_plan = 0; -- { serverError CANNOT_PARSE_TEXT } + +DROP TABLE t_se_outer; +DROP TABLE t_se_inner; +DROP TABLE t_se_local; diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference new file mode 100644 index 000000000000..f32381f38096 --- /dev/null +++ b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference @@ -0,0 +1,20 @@ +local +1 2 3 +2 3 4 +10 11 12 +merge_prefer0 +1 2 3 +2 3 4 +10 11 12 +merge_prefer1 +1 2 3 +2 3 4 +10 11 12 +merge_prefer0_plan +1 2 3 +2 3 4 +10 11 12 +merge_prefer1_plan +1 2 3 +2 3 4 +10 11 12 diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql new file mode 100644 index 000000000000..e1ac59428d5e --- /dev/null +++ b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql @@ -0,0 +1,58 @@ +-- Plain Merge over Distributed over MergeTree without an explicit __aliasMarker call. +-- Nested ALIAS columns (b contains a's subexpression). Reading the alias columns through the +-- Merge table must reconcile the child (Distributed) header by name; a positional reconciliation +-- in StorageMerge::convertAndFilterSourceStream would swap the columns (or fill them with 0). +-- The correct result equals the single-node ('local') result. +-- +-- Determinism notes: `x` is kept in GROUP BY so the ALIAS expansion can resolve it (the alias +-- expressions are defined in terms of x); GROUP BY also deduplicates the rows the two shards +-- produce, and ORDER BY x (distinct values) gives a total order independent of the distributed +-- merge order. So every block - local and the distributed variants - yields the same rows. +DROP TABLE IF EXISTS test_merge_alias_swap_merge; +DROP TABLE IF EXISTS test_merge_alias_swap_dist; +DROP TABLE IF EXISTS test_merge_alias_swap_local; + +CREATE TABLE test_merge_alias_swap_local +( + x UInt64, + a UInt64 ALIAS x + 1, + b UInt64 ALIAS a + 1 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_merge_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_merge_alias_swap_dist AS test_merge_alias_swap_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_merge_alias_swap_local); + +CREATE TABLE test_merge_alias_swap_merge +( + x UInt64, + a UInt64, + b UInt64 +) +ENGINE = Merge(currentDatabase(), '^test_merge_alias_swap_dist$'); + +SELECT 'local'; +SELECT x, a, b FROM test_merge_alias_swap_local GROUP BY x, a, b ORDER BY x; + +SELECT 'merge_prefer0'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1; + +SELECT 'merge_prefer0_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'merge_prefer1_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE test_merge_alias_swap_merge; +DROP TABLE test_merge_alias_swap_dist; +DROP TABLE test_merge_alias_swap_local; diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference new file mode 100644 index 000000000000..279f11b9bf19 --- /dev/null +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference @@ -0,0 +1,15 @@ +first +1999-03-29 01:15:33.000 +second +1999-03-29 01:15:33.000 +third +1999-03-29 01:15:33.000 +fourth +fifth +1999-03-29 01:15:33.000 +sixth +query_alias_0 query_alias_1 + +seventh +alias_String_7_0 alias_String_7_1 + diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql new file mode 100644 index 000000000000..8aa596998571 --- /dev/null +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql @@ -0,0 +1,67 @@ +-- Regression coverage for distributed ORDER BY + ALIAS columns with identical expressions. +-- Related issue: https://github.com/ClickHouse/ClickHouse/issues/79916 + +DROP TABLE IF EXISTS test_alias_same_expr_remote; + +CREATE TABLE test_alias_same_expr_remote +( + dt DateTime64(3), + String_7 String, + alias_String_7_0 String ALIAS String_7, + alias_String_7_1 String ALIAS String_7 +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_alias_same_expr_remote VALUES ('1999-03-29T01:15:33', ''); + +SELECT 'first'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +LIMIT 1; + +SELECT 'second'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 0; + +SELECT 'third'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1; + +SELECT 'fourth'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 0; -- { serverError NUMBER_OF_COLUMNS_DOESNT_MATCH } + +SELECT 'fifth'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1; + +SELECT 'sixth'; +SELECT alias_String_7_0 AS query_alias_0, alias_String_7_1 AS query_alias_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'seventh'; +SELECT alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +DROP TABLE test_alias_same_expr_remote; diff --git a/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference new file mode 100644 index 000000000000..8c2ebec53ff8 --- /dev/null +++ b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference @@ -0,0 +1,35 @@ +case1_global_in_unnamed_identical_derived_subqueries +1 +case2_global_join_unnamed_identical_derived_subqueries +id left_b0 right_b0 +1 10 20 +case3_global_join_unnamed_identical_derived_subqueries_serialize_query_plan +id left_b0 right_b0 +1 10 20 +case4_global_join_unnamed_remote_over_distributed_subqueries +id left_b0 right_b0 +1 10 20 +case5_global_join_unnamed_identical_dual_alias_columns +id left_b0 right_b1 +1 10 20 +case6_local_join_unnamed_identical_derived_subqueries +id left_b0 right_b0 +1 10 20 +case7_local_join_unnamed_identical_derived_subqueries_serialize_query_plan +id left_b0 right_b0 +1 10 20 +case8_global_join_direct_distributed_serialize_query_plan +id b0 b1 +1 10 10 +2 20 20 +case9_global_join_direct_remote_over_distributed_serialize_query_plan +id b0 b1 +1 10 10 +2 20 20 +case10_wrapper_alias_subquery_serialize_query_plan +id left_foo right_foo +1 1 20 +case11_wrapper_constant_alias_subquery_serialize_query_plan +id left_foo right_foo +1 foo foo +2 foo foo diff --git a/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql new file mode 100644 index 000000000000..2ac9a8a65fe3 --- /dev/null +++ b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql @@ -0,0 +1,297 @@ +DROP TABLE IF EXISTS test_marker_suite_main_dist; +DROP TABLE IF EXISTS test_marker_suite_side_dist; +DROP TABLE IF EXISTS test_marker_suite_main; +DROP TABLE IF EXISTS test_marker_suite_side; +DROP TABLE IF EXISTS test_wrapper_alias_a_dist; +DROP TABLE IF EXISTS test_wrapper_alias_b_dist; +DROP TABLE IF EXISTS test_wrapper_alias_a_local; +DROP TABLE IF EXISTS test_wrapper_alias_b_local; +DROP TABLE IF EXISTS test_wrapper_const_alias_a_dist; +DROP TABLE IF EXISTS test_wrapper_const_alias_b_dist; +DROP TABLE IF EXISTS test_wrapper_const_alias_a_local; +DROP TABLE IF EXISTS test_wrapper_const_alias_b_local; + +CREATE TABLE test_marker_suite_main +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_suite_main VALUES (1), (2); + +CREATE TABLE test_marker_suite_side +( + id UInt64, + x UInt64, + b0 UInt64 ALIAS x, + b1 UInt64 ALIAS x +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_suite_side VALUES (1, 10), (2, 20); + +CREATE TABLE test_marker_suite_main_dist AS test_marker_suite_main +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_suite_main, rand()); + +CREATE TABLE test_marker_suite_side_dist AS test_marker_suite_side +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_suite_side, rand()); + +SELECT 'case1_global_in_unnamed_identical_derived_subqueries'; +SELECT id +FROM test_marker_suite_main_dist +WHERE id GLOBAL IN +( + SELECT left_id + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) +ORDER BY id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1; + +SELECT 'case2_global_join_unnamed_identical_derived_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case3_global_join_unnamed_identical_derived_subqueries_serialize_query_plan'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case4_global_join_unnamed_remote_over_distributed_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist)) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist)) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case5_global_join_unnamed_identical_dual_alias_columns'; +SELECT m.id, j.left_b0, j.right_b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b1 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b1 AS right_b1 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b1 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case6_local_join_unnamed_identical_derived_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, distributed_product_mode = 'local' +FORMAT TSVWithNames; + +SELECT 'case7_local_join_unnamed_identical_derived_subqueries_serialize_query_plan'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, distributed_product_mode = 'local', serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case8_global_join_direct_distributed_serialize_query_plan'; +SELECT m.id, b0, b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN test_marker_suite_side_dist USING (id) +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, asterisk_include_alias_columns = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case9_global_join_direct_remote_over_distributed_serialize_query_plan'; +SELECT m.id, b0, b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist) USING (id) +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, asterisk_include_alias_columns = 1, joined_subquery_requires_alias = 0, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_marker_suite_main_dist; +DROP TABLE test_marker_suite_side_dist; +DROP TABLE test_marker_suite_main; +DROP TABLE test_marker_suite_side; + +CREATE TABLE test_wrapper_alias_a_local +( + id UInt64, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_wrapper_alias_b_local +( + id UInt64, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_wrapper_alias_a_local VALUES (1, 1), (2, 20); +INSERT INTO test_wrapper_alias_b_local VALUES (1, 1), (2, 20); + +CREATE TABLE test_wrapper_alias_a_dist +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_alias_a_local, rand()); + +CREATE TABLE test_wrapper_alias_b_dist +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_alias_b_local, rand()); + +SELECT 'case10_wrapper_alias_subquery_serialize_query_plan'; +SELECT a.id, j.left_foo, j.right_foo +FROM test_wrapper_alias_a_dist AS a +GLOBAL INNER JOIN +( + SELECT l.id, l.foo AS left_foo, r.foo AS right_foo + FROM test_wrapper_alias_a_dist AS l + INNER JOIN test_wrapper_alias_b_dist AS r ON l.id < r.id + WHERE l.foo + r.foo = 21 +) AS j +ON a.id = j.id +ORDER BY a.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_wrapper_alias_a_dist; +DROP TABLE test_wrapper_alias_b_dist; +DROP TABLE test_wrapper_alias_a_local; +DROP TABLE test_wrapper_alias_b_local; + +CREATE TABLE test_wrapper_const_alias_a_local +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_wrapper_const_alias_b_local +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_wrapper_const_alias_a_local VALUES (1), (2); +INSERT INTO test_wrapper_const_alias_b_local VALUES (1), (2); + +CREATE TABLE test_wrapper_const_alias_a_dist +( + id UInt64, + foo String ALIAS 'foo' +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_const_alias_a_local, rand()); + +CREATE TABLE test_wrapper_const_alias_b_dist +( + id UInt64, + foo String ALIAS 'foo' +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_const_alias_b_local, rand()); + +SELECT 'case11_wrapper_constant_alias_subquery_serialize_query_plan'; +SELECT a.id, j.left_foo, j.right_foo +FROM test_wrapper_const_alias_a_dist AS a +GLOBAL INNER JOIN +( + SELECT l.id, l.foo AS left_foo, r.foo AS right_foo + FROM test_wrapper_const_alias_a_dist AS l + INNER JOIN test_wrapper_const_alias_b_dist AS r ON l.id = r.id + WHERE l.foo = 'foo' AND r.foo = 'foo' +) AS j +ON a.id = j.id +ORDER BY a.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_wrapper_const_alias_a_dist; +DROP TABLE test_wrapper_const_alias_b_dist; +DROP TABLE test_wrapper_const_alias_a_local; +DROP TABLE test_wrapper_const_alias_b_local; diff --git a/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference new file mode 100644 index 000000000000..750abc85a605 --- /dev/null +++ b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference @@ -0,0 +1,56 @@ +prefer_localhost_replica_0 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_0_serialize_query_plan_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_1_serialize_query_plan_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 diff --git a/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql new file mode 100644 index 000000000000..e4bbceb7e6cf --- /dev/null +++ b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql @@ -0,0 +1,92 @@ +DROP TABLE IF EXISTS test_dod_double_alias_outer; +DROP TABLE IF EXISTS test_dod_double_alias_inner; +DROP TABLE IF EXISTS test_dod_double_alias_local; + +CREATE TABLE test_dod_double_alias_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_double_alias_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_double_alias_inner +( + x UInt64, + a UInt64 ALIAS 2, + b UInt64 ALIAS 2, + inner_c UInt64 ALIAS x + 1, + inner_d UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_double_alias_local); + +CREATE TABLE test_dod_double_alias_outer +( + x UInt64, + inner_c UInt64, + a UInt64 ALIAS 1, + b UInt64 ALIAS 1, + c UInt64 ALIAS inner_c, + d UInt64 ALIAS inner_c, + inner_d UInt64 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_double_alias_inner); + +SELECT 'prefer_localhost_replica_0'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_serialize_query_plan_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_serialize_query_plan_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_dod_double_alias_outer; +DROP TABLE test_dod_double_alias_inner; +DROP TABLE test_dod_double_alias_local; diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference new file mode 100644 index 000000000000..5155d27310c2 --- /dev/null +++ b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference @@ -0,0 +1,8 @@ +issue_1208_self_in_subquery +5 +issue_1209_join_mode_local +6 +issue_1209_join_mode_allow +6 +issue_1422_hybrid_in_merge_tree_subquery +5 diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql new file mode 100644 index 000000000000..c5cab11cee86 --- /dev/null +++ b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql @@ -0,0 +1,118 @@ +SET allow_experimental_hybrid_table = 1, + enable_analyzer = 1, + prefer_localhost_replica = 0, + -- AST-path regression test for unknown-table issues #1208/#1209/#1422. Pin + -- serialize_query_plan=0 so the "distributed plan" CI flavor (which forces it on) does not + -- route these hybrid + IN-subquery queries through the plan path, which has a separate, + -- unrelated header-reconciliation gap. + serialize_query_plan = 0; + +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_left; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_right; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_mt; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_left +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_right +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_mt +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +INSERT INTO test_hybrid_issue_1208_1209_1422_left VALUES + ('William', 9044, toDate('2024-01-01')), + ('Oliver', 1654, toDate('2024-01-01')), + ('Frank', 8751, toDate('2024-01-01')); + +INSERT INTO test_hybrid_issue_1208_1209_1422_right VALUES + ('Louis', 1519, toDate('2024-01-02')), + ('Isaac', 3611, toDate('2024-01-02')); + +INSERT INTO test_hybrid_issue_1208_1209_1422_mt +SELECT * FROM test_hybrid_issue_1208_1209_1422_left +UNION ALL +SELECT * FROM test_hybrid_issue_1208_1209_1422_right; + +CREATE TABLE test_hybrid_issue_1208_1209_1422 +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_left'), date_col <= '2024-01-01', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_right'), date_col > '2024-01-01' +); + +SELECT 'issue_1208_self_in_subquery'; +SELECT count() +FROM +( + SELECT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE string_col IN + ( + SELECT DISTINCT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE long_col > 1500 + ) +); + +SELECT 'issue_1209_join_mode_local'; +SELECT uniqExact(coalesce(h_string_col, m_string_col)) +FROM +( + SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long + FROM test_hybrid_issue_1208_1209_1422 AS h + FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col + SETTINGS object_storage_cluster_join_mode = 'local' +); + +SELECT 'issue_1209_join_mode_allow'; +SELECT uniqExact(coalesce(h_string_col, m_string_col)) +FROM +( + SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long + FROM test_hybrid_issue_1208_1209_1422 AS h + FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col + SETTINGS object_storage_cluster_join_mode = 'allow' +); + +SELECT 'issue_1422_hybrid_in_merge_tree_subquery'; +SELECT count() +FROM +( + SELECT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE string_col IN + ( + SELECT DISTINCT string_col + FROM test_hybrid_issue_1208_1209_1422_mt + WHERE long_col > 1500 + ) +); + +DROP TABLE test_hybrid_issue_1208_1209_1422; +DROP TABLE test_hybrid_issue_1208_1209_1422_left; +DROP TABLE test_hybrid_issue_1208_1209_1422_right; +DROP TABLE test_hybrid_issue_1208_1209_1422_mt; diff --git a/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference new file mode 100644 index 000000000000..f32381f38096 --- /dev/null +++ b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference @@ -0,0 +1,20 @@ +local +1 2 3 +2 3 4 +10 11 12 +merge_prefer0 +1 2 3 +2 3 4 +10 11 12 +merge_prefer1 +1 2 3 +2 3 4 +10 11 12 +merge_prefer0_plan +1 2 3 +2 3 4 +10 11 12 +merge_prefer1_plan +1 2 3 +2 3 4 +10 11 12 diff --git a/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql new file mode 100644 index 000000000000..c5817fd07f34 --- /dev/null +++ b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql @@ -0,0 +1,58 @@ +-- Plain Merge over Distributed over MergeTree (no Hybrid, no explicit __aliasMarker). +-- Nested ALIAS columns (b contains a's subexpression). Reading the alias columns through the +-- Merge table must reconcile the child (Distributed) header by name; a positional reconciliation +-- in StorageMerge::convertAndFilterSourceStream would swap the columns (or fill them with 0). +-- The correct result equals the single-node ('local') result. +-- +-- Determinism notes: `x` is kept in GROUP BY so the ALIAS expansion can resolve it (the alias +-- expressions are defined in terms of x); GROUP BY also deduplicates the rows the two shards +-- produce, and ORDER BY x (distinct values) gives a total order independent of the distributed +-- merge order. So every block - local and the distributed variants - yields the same rows. +DROP TABLE IF EXISTS test_merge_alias_swap_merge; +DROP TABLE IF EXISTS test_merge_alias_swap_dist; +DROP TABLE IF EXISTS test_merge_alias_swap_local; + +CREATE TABLE test_merge_alias_swap_local +( + x UInt64, + a UInt64 ALIAS x + 1, + b UInt64 ALIAS a + 1 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_merge_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_merge_alias_swap_dist AS test_merge_alias_swap_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_merge_alias_swap_local); + +CREATE TABLE test_merge_alias_swap_merge +( + x UInt64, + a UInt64, + b UInt64 +) +ENGINE = Merge(currentDatabase(), '^test_merge_alias_swap_dist$'); + +SELECT 'local'; +SELECT x, a, b FROM test_merge_alias_swap_local GROUP BY x, a, b ORDER BY x; + +SELECT 'merge_prefer0'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1; + +SELECT 'merge_prefer0_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'merge_prefer1_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE test_merge_alias_swap_merge; +DROP TABLE test_merge_alias_swap_dist; +DROP TABLE test_merge_alias_swap_local; diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference new file mode 100644 index 000000000000..402cc360bae5 --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference @@ -0,0 +1,15 @@ +local +11 12 +21 22 +dist_prefer0 +11 12 +21 22 +dist_prefer1 +11 12 +21 22 +dist_prefer0_plan +11 12 +21 22 +dist_prefer1_plan +11 12 +21 22 diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql new file mode 100644 index 000000000000..848f35b0be14 --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql @@ -0,0 +1,34 @@ +-- Plain Distributed (no Hybrid). Two nested ALIAS columns: a2 contains a1's subexpression, +-- so planner CSE may reorder the remote header. Correct result must equal the single-node +-- ('local') result across every transport variant. +DROP TABLE IF EXISTS t_local_03930; +DROP TABLE IF EXISTS t_dist_03930; + +CREATE TABLE t_local_03930 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03930 VALUES (10), (20); + +CREATE TABLE t_dist_03930 AS t_local_03930 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03930); + +SELECT 'local'; +SELECT a1, a2 FROM t_local_03930 ORDER BY a1; + +SELECT 'dist_prefer0'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'dist_prefer1'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1; + +SELECT 'dist_prefer0_plan'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'dist_prefer1_plan'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE t_dist_03930; +DROP TABLE t_local_03930; diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference new file mode 100644 index 000000000000..97dd73615024 --- /dev/null +++ b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference @@ -0,0 +1,9 @@ +local +1 10 100 +2 20 200 +merge_prefer0 +1 10 100 +2 20 200 +merge_prefer1 +1 10 100 +2 20 200 diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql new file mode 100644 index 000000000000..c42ad63adab4 --- /dev/null +++ b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql @@ -0,0 +1,64 @@ +-- Regression for the StorageMerge alias-output-naming fix. +-- The bug: `Nested::splitName(name, reverse=true)` (used before this fix to strip the +-- analyzer's `__tableN.` prefix from header column names) splits on the LAST dot, so for +-- an analyzer identifier like `__table1.\`n.a\`` (a dotted column name wrapped in backticks +-- by the analyzer) it returns the suffix `a\`` instead of `n.a`, leaving the +-- `logical_name_to_header_name` map with broken keys. The lookup for `alias.name == "n.a"` +-- then misses, the alias output falls back to the bare name `n.a`, and the downstream +-- header-reconciliation step fills the expected `__table1.\`n.a\`` column with type +-- defaults (zeros). Silent wrong data. +-- +-- Repro shape: Merge declares dotted column names explicitly (typical when matching a +-- schema with Nested-style names), underlying storage has those columns as ALIAS, and +-- the Distributed routing forces analyzer-prefixed names in the Merge level. Using a +-- two-shard cluster with prefer_localhost_replica=0 reliably reproduces. + +DROP TABLE IF EXISTS test_04286_dotted_alias_local; +DROP TABLE IF EXISTS test_04286_dotted_alias_dist; +DROP TABLE IF EXISTS test_04286_dotted_alias_merge; + +CREATE TABLE test_04286_dotted_alias_local +( + id UInt32, + `n.a` UInt32 ALIAS id * 10, + `m.b` UInt32 ALIAS id * 100 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO test_04286_dotted_alias_local VALUES (1), (2); + +CREATE TABLE test_04286_dotted_alias_dist AS test_04286_dotted_alias_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_04286_dotted_alias_local); + +CREATE TABLE test_04286_dotted_alias_merge +( + id UInt32, + `n.a` UInt32, + `m.b` UInt32 +) +ENGINE = Merge(currentDatabase(), '^test_04286_dotted_alias_dist$'); + +SELECT 'local'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_local +GROUP BY id, `n.a`, `m.b` +ORDER BY id; + +SELECT 'merge_prefer0'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_merge +GROUP BY id, `n.a`, `m.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_merge +GROUP BY id, `n.a`, `m.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 1; + +DROP TABLE test_04286_dotted_alias_merge; +DROP TABLE test_04286_dotted_alias_dist; +DROP TABLE test_04286_dotted_alias_local; diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference new file mode 100644 index 000000000000..dcebc31194b1 --- /dev/null +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference @@ -0,0 +1,30 @@ +merge_tree_row_count +5 +iceberg_row_count +5 +hybrid_row_count +5 +true 8751 7291.267979503492 Frank 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 5313 8428.52 456.78 +false 3611 4492.090462838536 Isaac 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 4552 1554.795 456.78 +true 1519 3799.273006373374 Louis 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 8785 1248.2616 456.78 +true 1654 3801.2622503916614 Oliver 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3432 6701.752 456.78 +true 9044 2931.782814070929 William 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3733 7730.6836 456.78 +issue_1208_join_hybrid_mt_local +Frank 8751 8751 +Isaac 3611 3611 +Louis 1519 1519 +Oliver 1654 1654 +William 9044 9044 +issue_1208_join_hybrid_mt_allow +Frank 8751 8751 +Isaac 3611 3611 +Louis 1519 1519 +Oliver 1654 1654 +William 9044 9044 +issue_1208_join_hybrid_mt_iceberg_local +issue_1208_join_hybrid_mt_iceberg_allow +Frank 8751 8751 8751 +Isaac 3611 3611 3611 +Louis 1519 1519 1519 +Oliver 1654 1654 1654 +William 9044 9044 9044 diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql new file mode 100644 index 000000000000..a37c991684b4 --- /dev/null +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql @@ -0,0 +1,329 @@ +SET allow_experimental_hybrid_table = 1, + enable_analyzer = 1, + prefer_localhost_replica = 0, + iceberg_delete_data_on_drop = 1; + +DROP TABLE IF EXISTS hybrid_table; +DROP TABLE IF EXISTS merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; +DROP TABLE IF EXISTS iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; +DROP TABLE IF EXISTS merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; +DROP TABLE IF EXISTS hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; +DROP TABLE IF EXISTS merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; +DROP TABLE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; +DROP DATABASE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172; + +CREATE TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 VALUES + (true, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), + (true, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), + (true, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), + (true, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), + (false, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); + +SELECT 'merge_tree_row_count'; +SELECT count() FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; + +CREATE TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 +( + boolean_col Nullable(Int32), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Float64) +) +ENGINE = IcebergS3( + s3_conn, + filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table') +); + +INSERT INTO iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 VALUES + (1, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), + (1, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), + (1, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), + (1, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), + (0, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); + +SELECT 'iceberg_row_count'; +SELECT count() FROM iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; + +CREATE TABLE hybrid_table +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = Hybrid( + remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), + date_col <= '2024-01-01', + icebergCluster( + 'test_cluster_one_shard_three_replicas_localhost', + concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), + 'test', + 'testtest' + ), + date_col > '2024-01-01' +); + +SELECT 'hybrid_row_count'; +SELECT count() FROM hybrid_table; + +CREATE TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 +SELECT * FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; + +CREATE DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; + +CREATE TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` +( + boolean_col Nullable(Int32), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Float64) +) +ENGINE = IcebergS3( + s3_conn, + filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table_39afe20a_d5d6_11f0_8208_e0c26496f172') +); + +INSERT INTO database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` +SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 +SELECT + toInt32(boolean_col), + long_col, + double_col, + string_col, + timestamp_col, + date_col, + time_col, + timestamptz_col, + integer_col, + float_col, + toFloat64(decimal_col) +FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; + +SELECT * +FROM hybrid_table +WHERE string_col IN +( + SELECT DISTINCT string_col + FROM hybrid_table + WHERE long_col > 1500 +) +ORDER BY string_col; + +SELECT 'issue_1208_join_hybrid_mt_local'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local'; + +SELECT 'issue_1208_join_hybrid_mt_allow'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow'; + +SELECT 'issue_1208_join_hybrid_mt_iceberg_local'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local'; -- { serverError UNKNOWN_IDENTIFIER } + +SELECT 'issue_1208_join_hybrid_mt_iceberg_allow'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow'; + +-- Exact issue-shape queries (no ORDER BY), deterministic output via FORMAT Null. +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local' +FORMAT Null; + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow' +FORMAT Null; + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local' +FORMAT Null; -- { serverError UNKNOWN_IDENTIFIER } + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow' +FORMAT Null; + +CREATE TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 +SELECT * FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; + +CREATE TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = Hybrid( + remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), + date_col <= '2024-01-01', + icebergCluster( + 'test_cluster_one_shard_three_replicas_localhost', + concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), + 'test', + 'testtest' + ), + date_col > '2024-01-01' +); + +SELECT * +FROM hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 +WHERE string_col IN +( + SELECT DISTINCT string_col + FROM merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 + WHERE long_col > 1500 +) +FORMAT Null; + +DROP TABLE hybrid_table; +DROP TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; +DROP TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; +DROP TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; +DROP TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; +DROP TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; +DROP TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; +DROP DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference new file mode 100644 index 000000000000..1e2e9b11750a --- /dev/null +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference @@ -0,0 +1,56 @@ +prefer_localhost_replica_0_uint64 +x a_num inner_c +1 1 2 +1 1 2 +1 1 2 +1 1 2 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +10 1 11 +10 1 11 +10 1 11 +10 1 11 +prefer_localhost_replica_0_string +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +prefer_localhost_replica_1_uint64 +x a_num inner_c +1 1 2 +1 1 2 +1 1 2 +1 1 2 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +10 1 11 +10 1 11 +10 1 11 +10 1 11 +prefer_localhost_replica_1_string +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql new file mode 100644 index 000000000000..84ad3cf170d0 --- /dev/null +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql @@ -0,0 +1,96 @@ +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_outer; +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_inner; +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_local; + +CREATE TABLE test_dod_alias_swap_no_marker_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_alias_swap_no_marker_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_alias_swap_no_marker_inner +( + x UInt64, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_no_marker_local); + +CREATE TABLE test_dod_alias_swap_no_marker_outer +( + x UInt64, + inner_c UInt64, + a_num UInt64 ALIAS 1, + a_str String ALIAS 'aaaa' +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_no_marker_inner); + +SELECT 'prefer_localhost_replica_0_uint64'; +SELECT + x, + a_num, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_string'; +SELECT + x, + a_str, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_uint64'; +SELECT + x, + a_num, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_string'; +SELECT + x, + a_str, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +DROP TABLE test_dod_alias_swap_no_marker_outer; +DROP TABLE test_dod_alias_swap_no_marker_inner; +DROP TABLE test_dod_alias_swap_no_marker_local; diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference new file mode 100644 index 000000000000..228ac5f667f7 --- /dev/null +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference @@ -0,0 +1,20 @@ +no_pr_uint64 +x a_num inner_c +1 1 2 +2 1 3 +10 1 11 +no_pr_string +x a_str inner_c +1 aaaa 2 +2 aaaa 3 +10 aaaa 11 +pr_uint64 +x a_num inner_c +1 1 2 +2 1 3 +10 1 11 +pr_string +x a_str inner_c +1 aaaa 2 +2 aaaa 3 +10 aaaa 11 diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql new file mode 100644 index 000000000000..070330e98826 --- /dev/null +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql @@ -0,0 +1,94 @@ +DROP TABLE IF EXISTS test_pr_dod_alias_swap_outer; +DROP TABLE IF EXISTS test_pr_dod_alias_swap_inner; +DROP TABLE IF EXISTS test_pr_dod_alias_swap_local; + +CREATE TABLE test_pr_dod_alias_swap_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_pr_dod_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_pr_dod_alias_swap_inner +( + x UInt64, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_pr_dod_alias_swap_local); + +CREATE TABLE test_pr_dod_alias_swap_outer +( + x UInt64, + inner_c UInt64, + a_num UInt64 ALIAS 1, + a_str String ALIAS 'aaaa' +) +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_pr_dod_alias_swap_inner); + +SELECT 'no_pr_uint64'; +SELECT x, a_num, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 0, + allow_experimental_parallel_reading_from_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + parallel_replicas_for_non_replicated_merge_tree = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'no_pr_string'; +SELECT x, a_str, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 0, + allow_experimental_parallel_reading_from_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + parallel_replicas_for_non_replicated_merge_tree = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'pr_uint64'; +SELECT x, a_num, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 2, + allow_experimental_parallel_reading_from_replicas = 2, + max_parallel_replicas = 3, + parallel_replicas_local_plan = 1, + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_min_number_of_rows_per_replica = 0, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'pr_string'; +SELECT x, a_str, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 2, + allow_experimental_parallel_reading_from_replicas = 2, + max_parallel_replicas = 3, + parallel_replicas_local_plan = 1, + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_min_number_of_rows_per_replica = 0, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +DROP TABLE test_pr_dod_alias_swap_outer; +DROP TABLE test_pr_dod_alias_swap_inner; +DROP TABLE test_pr_dod_alias_swap_local; diff --git a/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference new file mode 100644 index 000000000000..f3797cb0ce0e --- /dev/null +++ b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference @@ -0,0 +1,32 @@ +prefer_localhost_replica_0_uint64 +a inner_c +1 2 +1 2 +1 3 +1 3 +1 11 +1 11 +prefer_localhost_replica_0_string +a inner_c +aaaa 2 +aaaa 2 +aaaa 3 +aaaa 3 +aaaa 11 +aaaa 11 +prefer_localhost_replica_1_uint64 +a inner_c +1 2 +1 2 +1 3 +1 3 +1 11 +1 11 +prefer_localhost_replica_1_string +a inner_c +aaaa 2 +aaaa 2 +aaaa 3 +aaaa 3 +aaaa 11 +aaaa 11 diff --git a/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql new file mode 100644 index 000000000000..50085531b6dd --- /dev/null +++ b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql @@ -0,0 +1,82 @@ +DROP TABLE IF EXISTS test_dod_alias_swap_local; +DROP TABLE IF EXISTS test_dod_alias_swap_inner; + +CREATE TABLE test_dod_alias_swap_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_alias_swap_inner +( + x UInt64, + a UInt64 ALIAS 2, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_local); + +SELECT 'prefer_localhost_replica_0_uint64'; +SELECT + __aliasMarker(_CAST(1, 'UInt64'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_string'; +SELECT + __aliasMarker(_CAST('aaaa', 'String'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_uint64'; +SELECT + __aliasMarker(_CAST(1, 'UInt64'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_string'; +SELECT + __aliasMarker(_CAST('aaaa', 'String'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +DROP TABLE test_dod_alias_swap_inner; +DROP TABLE test_dod_alias_swap_local; From cbebdb3aa7c5179d6fde05c03d42f8c8f7eda6a5 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 15 Sep 2026 12:49:53 +0200 Subject: [PATCH 02/18] Fix MULTIPLE_EXPRESSIONS_FOR_ALIAS from __aliasMarker on distributed ALIAS columns A `Distributed` query that reads the same-named `ALIAS` column from two `JOIN` sources fails on the shard: Multiple expressions __aliasMarker(__table2.x, '__table4.foo') AS foo and __aliasMarker(__table1.x, '__table3.foo') AS foo for alias foo `ReplaseAliasColumnsVisitor` stamped `setAlias(column_name)` on every marker it created, so two markers carrying different ids both ended up named `foo` in the same scope. It also aliased `column_node->getExpression()` in place, which is shared between occurrences, so aliasing one occurrence changed the others. Set the alias only where it is read. A top-level projection item keeps it so the mergeable-state output column keeps its name, and a `JOIN USING` key side keeps it because the shipped SQL renders the side rather than the key and `rejectUnshippableJoinUsingKeys` reads that shape. Everywhere else -- `WHERE`, `GROUP BY`, `ORDER BY`, `HAVING`, `JOIN ON` -- no alias is set, which is what removes the collision. Clone the defining expression per occurrence so setting an alias on one no longer reaches another. The pass moves from `StorageDistributed` to `buildQueryTreeForShard` as `inlineAliasColumns`, matching the name, file and semantics of the shared inliner upstream introduced in https://github.com/ClickHouse/ClickHouse/pull/107700, which fixes the same class of failure (https://github.com/ClickHouse/ClickHouse/issues/107990) without markers. Writing it there keeps the next version bump a merge rather than a rewrite. The signature takes a `ContextPtr` that upstream's does not, because marker injection needs the `enable_alias_marker` setting and function resolution; with the setting off, the pass behaves exactly as upstream's. Not ported here: the parallel-replicas call sites #107700 adds. No staged test needs them yet. Verified against the 20 staged regression tests from https://github.com/Altinity/ClickHouse/pull/1844: `05054_distributed_global_alias_marker_matrix` now passes, and the other 19 are byte-identical to the run before this change -- same error codes, same messages. `05061` covers `enable_alias_marker = 0`, and `03844` / `05055` cover nested marker chains; all three still pass. Related: https://github.com/Altinity/ClickHouse/pull/1844 Related: https://github.com/ClickHouse/ClickHouse/pull/107700 Co-Authored-By: Claude Opus 5 --- src/Storages/StorageDistributed.cpp | 75 +-------- src/Storages/buildQueryTreeForShard.cpp | 205 ++++++++++++++++++++++++ src/Storages/buildQueryTreeForShard.h | 17 ++ 3 files changed, 224 insertions(+), 73 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index a51044cfe76d..fe73ea18b98d 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -55,7 +54,6 @@ #include #include -#include #include #include #include @@ -213,7 +211,6 @@ namespace Setting extern const SettingsBool skip_unavailable_shards; extern const SettingsBool enable_global_with_statement; extern const SettingsBool allow_experimental_hybrid_table; - extern const SettingsBool enable_alias_marker; } namespace DistributedSetting @@ -849,73 +846,6 @@ StorageSnapshotPtr StorageDistributed::getStorageSnapshot(const StorageMetadataP namespace { -class ReplaseAliasColumnsVisitor : public InDepthQueryTreeVisitor -{ - QueryTreeNodePtr getColumnNodeAliasExpression(const QueryTreeNodePtr & node) const - { - const auto * column_node = node->as(); - if (!column_node || !column_node->hasExpression()) - return nullptr; - - const auto & column_source = column_node->getColumnSourceOrNull(); - if (!column_source || column_source->getNodeType() == QueryTreeNodeType::JOIN - || column_source->getNodeType() == QueryTreeNodeType::CROSS_JOIN - || column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) - return nullptr; - - auto column_expression = column_node->getExpression(); - const auto & column_name = column_node->getColumnName(); - - if (!context->getSettingsRef()[Setting::enable_alias_marker]) - { - column_expression->setAlias(column_name); - return column_expression; - } - - String alias_id; - const auto & source_alias = column_source->getAlias(); - if (!source_alias.empty()) - alias_id = source_alias + "." + column_name; - else - alias_id = column_name; - - if (auto * function_node = column_expression->as(); - function_node && function_node->getFunctionName() == "__aliasMarker") - { - auto & arguments = function_node->getArguments().getNodes(); - if (arguments.size() == 2) - arguments[1] = std::make_shared(alias_id, std::make_shared()); - - column_expression->setAlias(column_name); - return column_expression; - } - - QueryTreeNodes arguments; - arguments.reserve(2); - arguments.emplace_back(std::move(column_expression)); - arguments.emplace_back(std::make_shared(alias_id, std::make_shared())); - - auto alias_marker_node = std::make_shared("__aliasMarker"); - alias_marker_node->getArguments().getNodes() = std::move(arguments); - alias_marker_node->setAlias(column_name); - resolveOrdinaryFunctionNodeByName(*alias_marker_node, "__aliasMarker", context); - - return alias_marker_node; - } - -public: - explicit ReplaseAliasColumnsVisitor(ContextPtr context_) : context(std::move(context_)) {} - - void visitImpl(QueryTreeNodePtr & node) - { - if (auto column_expression = getColumnNodeAliasExpression(node)) - node = column_expression; - } - -private: - ContextPtr context; -}; - using ColumnNameToColumnNodeMap = std::unordered_map; ColumnNameToColumnNodeMap buildColumnNodesForTableExpression(const QueryTreeNodePtr & table_expression_node, const ContextPtr & context) @@ -1151,7 +1081,7 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, * (including fully-resolved ALIAS expressions) and rewrite the whole query tree * so all references to the replaced table share the same column source and * the same alias semantics. This keeps SELECT and WHERE consistent before - * ReplaseAliasColumnsVisitor performs final alias expansion. + * inlineAliasColumns performs final alias expansion. */ ReplaceColumnNodesForTableExpressionVisitor replace_query_columns_visitor( replacement_table_expression, @@ -1160,8 +1090,7 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, replace_query_columns_visitor.visit(query_tree_to_modify); } - ReplaseAliasColumnsVisitor replace_alias_columns_visitor(query_context); - replace_alias_columns_visitor.visit(query_tree_to_modify); + inlineAliasColumns(query_tree_to_modify, query_context); const auto & settings = query_context->getSettingsRef(); diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 9669cd40ad7e..eb1374aad67d 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -62,6 +62,7 @@ namespace Setting extern const SettingsBool parallel_replicas_prefer_local_join; extern const SettingsBool prefer_global_in_and_join; extern const SettingsBool enable_add_distinct_to_in_subqueries; + extern const SettingsBool enable_alias_marker; extern const SettingsInt64 optimize_const_name_size; extern const SettingsOverflowMode transfer_overflow_mode; extern const SettingsObjectStorageClusterJoinMode object_storage_cluster_join_mode; @@ -78,6 +79,204 @@ namespace ErrorCodes namespace { +/// Return a clone of the defining expression of an inlineable `ALIAS` column node, or nullptr otherwise. +/// A JOIN / CROSS_JOIN / ARRAY_JOIN source puts a `ListNode` of the joined sides in the expression child, which is not +/// an alias body. The expression is cloned so each occurrence gets its own copy: that lets one occurrence be aliased +/// (a projection output) without mutating another (a reference in `ORDER BY`). +QueryTreeNodePtr getInlineableAliasColumnExpression(const QueryTreeNodePtr & node) +{ + const auto * column_node = node->as(); + if (!column_node || !column_node->hasExpression()) + return nullptr; + + const auto & column_source = column_node->getColumnSourceOrNull(); + if (!column_source || column_source->getNodeType() == QueryTreeNodeType::JOIN + || column_source->getNodeType() == QueryTreeNodeType::CROSS_JOIN + || column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) + return nullptr; + + return column_node->getExpression()->clone(); +} + +/// The id a marker carries for `column_node`. The source's alias qualifies the column name whenever the source has one, +/// so two same-named `ALIAS` columns coming from different tables stay distinguishable in the shard header. +String aliasMarkerIdFor(const ColumnNode & column_node) +{ + const auto & column_source = column_node.getColumnSourceOrNull(); + const auto & source_alias = column_source ? column_source->getAlias() : String{}; + if (source_alias.empty()) + return column_node.getColumnName(); + return source_alias + "." + column_node.getColumnName(); +} + +/// Wrap an inlined `ALIAS` expression in `__aliasMarker` so it carries the identity of the column it came from. +/// An expression that is already a marker gets its id refreshed rather than a second wrapper around it. +QueryTreeNodePtr wrapInAliasMarker(QueryTreeNodePtr expression, const ColumnNode & column_node, const ContextPtr & context) +{ + auto alias_id = aliasMarkerIdFor(column_node); + + if (auto * function_node = expression->as(); + function_node && function_node->getFunctionName() == "__aliasMarker") + { + auto & marker_arguments = function_node->getArguments().getNodes(); + if (marker_arguments.size() == 2) + marker_arguments[1] = std::make_shared(std::move(alias_id), std::make_shared()); + return expression; + } + + QueryTreeNodes arguments; + arguments.reserve(2); + arguments.emplace_back(std::move(expression)); + arguments.emplace_back(std::make_shared(std::move(alias_id), std::make_shared())); + + auto marker_node = std::make_shared("__aliasMarker"); + marker_node->getArguments().getNodes() = std::move(arguments); + resolveOrdinaryFunctionNodeByName(*marker_node, "__aliasMarker", context); + + return marker_node; +} + +/// Inlines `ALIAS` columns into their defining expressions across a query tree that is about to be shipped. +/// See `inlineAliasColumns` in the header for what the pass guarantees and why. +struct AliasColumnInliner +{ + ContextPtr context; + bool use_alias_marker; + + /// Replace one inlineable `ALIAS` column node with its body, marking the body when markers are on. + /// Returns false when `node` is not an inlineable `ALIAS` column, leaving it untouched. + bool inlineOnce(QueryTreeNodePtr & node) const + { + auto expression = getInlineableAliasColumnExpression(node); + if (!expression) + return false; + + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), node->as(), context); + + node = std::move(expression); + return true; + } + + /// Inline `ALIAS` columns inside an expression subtree without assigning any alias. A nested subquery goes back + /// through `inlineQuery` so its own projection columns keep their names. + void inlineExpression(QueryTreeNodePtr & node) const + { + if (node->as() || node->as()) + { + inlineQuery(node); + return; + } + + /// An `ALIAS` column may be defined over another one, so keep unwrapping. With markers on the loop stops after + /// the first step, because the wrapper is a function node; the column it wrapped is reached through the + /// children below and gets its own marker there, which is what preserves a nested marker chain. + while (inlineOnce(node)) + { + } + + auto * join_node = node->as(); + const bool using_join = join_node && join_node->isUsingJoinExpression(); + + for (auto & child : node->getChildren()) + { + if (!child) + continue; + + if (using_join && child == join_node->getJoinExpression()) + inlineJoinUsingKeys(child); + else + inlineExpression(child); + } + } + + /// A `JOIN USING` key is a `ColumnNode` whose expression is a `ListNode` recording how the key resolves on each + /// side, and a side's entry can itself be an `ALIAS` column. Such an entry keeps the key's name as an alias when + /// inlined, because the shipped SQL renders the entry rather than the key: `USING (x AS a)` is what lets a remote + /// server resolve a key that exists only as an `ALIAS` column of the initiator's table. `rejectUnshippableJoinUsingKeys` + /// reads the same shape to decide which keys no remote server can resolve. + void inlineJoinUsingKeys(QueryTreeNodePtr & join_expression) const + { + auto * using_list = join_expression->as(); + if (!using_list) + return; + + for (auto & using_node : using_list->getNodes()) + { + auto * using_column = using_node->as(); + if (!using_column || !using_column->hasExpression()) + continue; + + auto * key_sides = using_column->getExpression()->as(); + if (!key_sides) + continue; + + for (auto & side : key_sides->getNodes()) + { + const auto * side_column = side->as(); + auto expression = getInlineableAliasColumnExpression(side); + if (!expression) + continue; + + const String key_name = side_column->getColumnName(); + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), *side_column, context); + + inlineExpression(expression); + expression->setAlias(key_name); + side = std::move(expression); + } + } + } + + /// Inline `ALIAS` columns into their defining expressions, so the expression is evaluated on the shard that reads + /// the real table instead of the column being resolved there as if it were physical. + /// + /// A top-level projection item keeps the column's logical name as an alias, so the mergeable-state output column + /// keeps its name. Inside expression clauses no alias is set: otherwise two same-named `ALIAS` columns from + /// different `JOIN` sources land in one scope carrying different bodies, and the shard throws + /// `MULTIPLE_EXPRESSIONS_FOR_ALIAS` (https://github.com/ClickHouse/ClickHouse/issues/107990). + void inlineQuery(QueryTreeNodePtr & node) const + { + if (auto * union_node = node->as()) + { + for (auto & query : union_node->getQueries().getNodes()) + inlineQuery(query); + return; + } + + auto * query_node = node->as(); + if (!query_node) + { + inlineExpression(node); + return; + } + + for (auto & projection_item : query_node->getProjection().getNodes()) + { + const auto * column_node = projection_item->as(); + auto expression = getInlineableAliasColumnExpression(projection_item); + if (!expression) + { + inlineExpression(projection_item); + continue; + } + + const String output_alias = column_node->getColumnName(); + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), *column_node, context); + + inlineExpression(expression); + expression->setAlias(output_alias); + projection_item = std::move(expression); + } + + for (auto & child : query_node->getChildren()) + if (child && child != query_node->getProjectionNode()) + inlineExpression(child); + } +}; + /// Visitor that collect column source to columns mapping from query and all subqueries class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitor { @@ -738,6 +937,12 @@ void rejectUnshippableJoinUsingKeys(const QueryTreeNodePtr & root) } +void inlineAliasColumns(QueryTreeNodePtr & query_tree_to_modify, const ContextPtr & context) +{ + const AliasColumnInliner inliner{context, context->getSettingsRef()[Setting::enable_alias_marker]}; + inliner.inlineQuery(query_tree_to_modify); +} + QueryTreeNodePtr buildQueryTreeForShard( const PlannerContextPtr & planner_context, QueryTreeNodePtr query_tree_to_modify, diff --git a/src/Storages/buildQueryTreeForShard.h b/src/Storages/buildQueryTreeForShard.h index 5e3ae678dcf6..7730c43adc6b 100644 --- a/src/Storages/buildQueryTreeForShard.h +++ b/src/Storages/buildQueryTreeForShard.h @@ -27,6 +27,23 @@ QueryTreeNodePtr buildQueryTreeForShard( bool allow_global_join_for_right_table, bool find_cross_join = false); +/** Replace every `ALIAS` column node with its defining expression, so the expression is evaluated on the shard that reads + * the real table instead of the column being resolved there as if it were physical. + * + * Apply this to any query tree that is about to be shipped, before `buildQueryTreeForShard`: that function rebuilds a + * shipped table expression from column names and types only, which drops an `ALIAS` column's resolved expression and + * leaves the remote side asking storage for a column it does not have. + * + * When `enable_alias_marker` is on, each inlined expression is also wrapped in `__aliasMarker(expr, '')`. The + * marker preserves the identity of the logical column the expression was expanded from, so the initiator -- which + * still sees the un-inlined column -- can match the shard header by name instead of by position. + * + * The column's logical name is kept as an alias on top-level projection items and on `JOIN USING` key sides only. + * Aliasing an occurrence inside `WHERE` / `GROUP BY` / `ORDER BY` / `HAVING` / `JOIN ON` makes two same-named `ALIAS` + * columns from different sources collide in one scope, and the shard then throws `MULTIPLE_EXPRESSIONS_FOR_ALIAS`. + */ +void inlineAliasColumns(QueryTreeNodePtr & query_tree_to_modify, const ContextPtr & context); + void rewriteJoinToGlobalJoin(QueryTreeNodePtr query_tree_to_modify, ContextPtr context, bool force_prefer_global_join = false); void rewriteInToGlobalIn(QueryTreeNodePtr & query_tree_to_modify, ContextPtr context, bool rewrite_for_distributed = false); From 1796c7fe359401e08e04a9ff690811752c48e371 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 15 Sep 2026 16:23:35 +0200 Subject: [PATCH 03/18] Fix ALIAS columns read through Merge over Distributed Reading an `ALIAS` column from a `Merge` table whose child is `Distributed` fails on the initiator: Not found column x: in block __table1.x UInt64, plus(__table1.x, 1_UInt8) UInt64 `ReadFromMerge::convertAndFilterSourceStream` recomputes alias values at the Merge level and emitted each one under its plain logical name, while the Merge target header names its columns with analyzer identifiers. The later `addMissingDefaults` then found no `__tableN.` column and filled it with type defaults. The final header reconciliation matched by position, so a child that returned the same columns in a different order -- which a `Distributed` child does, because it inlines alias expressions on the shard and returns them in whatever order the shard's `ActionsDAG` produced -- silently paired values with the wrong columns. Emit each alias output under its analyzer identifier instead, and reconcile the child header by name. The plain-name to identifier mapping is built by matching each column declared on the Merge table against the suffixes of the child header names, rather than by pattern-matching the analyzer's `__tableN.` naming convention. Driving it from the declared schema is what makes dotted names work: a `Nested` subcolumn arrives as ``__table1.`n.a` `` on some branches and `__table1.n.a` on others, and splitting on the last dot mangles both. Alias expressions name their inputs in plain logical names, so every mapped column is also exposed under its plain name in the per-alias `ActionsDAG`. An `ALIAS` defined over another `ALIAS` needs that to resolve. `makeConvertingActionsPreferNameThenPosition` in `Planner/Utils` performs the reconciliation. It matches by name only when both headers carry the same set of names with no duplicates, which makes the rename a pure reordering, and matches by position otherwise. The position branch is load-bearing rather than a safety net: a source header can legitimately carry names the result never mentions, such as the synthetic aggregate-state column behind an optimized trivial `count()`. A `LOG_TEST` line reports which call site fell back and why. `convertAndFilterSourceStream` takes the Merge table's `ColumnsDescription` for this; the planner's `TableExpressionData` is not populated yet at that point, so the mapping cannot be looked up through `PlannerContext`. The `TODO(storage-merge-alias)` comment records why this recomputation exists at all and what would remove it. Its two steps are internal to `StorageMerge` and unrelated to the commits around it. Verified against the 20 staged regression tests from https://github.com/Altinity/ClickHouse/pull/1844: `04281_storage_merge_over_distributed_alias`, `05057_merge_over_distributed_alias_marker_column_swap` and `05059_dotted_alias_merge_over_distributed` now pass, and the four remaining failures are byte-identical to the previous run. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- src/Planner/Utils.cpp | 61 +++++++++++++++ src/Planner/Utils.h | 22 ++++++ src/Storages/StorageMerge.cpp | 140 ++++++++++++++++++++++++++++++++-- src/Storages/StorageMerge.h | 1 + 4 files changed, 218 insertions(+), 6 deletions(-) diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index f48515204da2..e375619925e2 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -1,5 +1,8 @@ #include +#include +#include + #include #include #include @@ -847,4 +850,62 @@ QueryPlanStepPtr projectOnlyUsedColumns( return step; } +namespace +{ + +/// True when both sides hold the same set of names and neither repeats one, so matching by name is a pure reordering. +bool canMatchByNameWithoutAmbiguity(const ColumnsWithTypeAndName & source, const ColumnsWithTypeAndName & result) +{ + if (source.size() != result.size()) + return false; + + NameSet source_names; + NameSet result_names; + + for (const auto & source_column : source) + if (!source_names.insert(source_column.name).second) + return false; + + for (const auto & result_column : result) + if (!result_names.insert(result_column.name).second) + return false; + + return source_names == result_names; +} + +} + +ActionsDAG makeConvertingActionsPreferNameThenPosition( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & result_columns, + const ContextPtr & context, + std::string_view location, + bool ignore_constant_values, + bool add_cast_columns, + NameToNameMap * new_names) +{ + const bool match_by_name = canMatchByNameWithoutAmbiguity(source_columns, result_columns); + + if (!match_by_name) + { + /// `LOG_IMPL` checks the level before formatting, so the two `dumpNames` calls stay off the hot path. + static const auto log = getLogger("Planner"); + LOG_TEST( + log, + "Matching by position at {}, because the names do not line up as a set: source=[{}] result=[{}]", + location, + Block(source_columns).dumpNames(), + Block(result_columns).dumpNames()); + } + + return ActionsDAG::makeConvertingActions( + source_columns, + result_columns, + match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, + context, + ignore_constant_values, + add_cast_columns, + new_names); +} + } diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 97385e9f56fc..10858dc94e3a 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -10,6 +10,8 @@ #include #include + +#include #include #include @@ -124,4 +126,24 @@ QueryPlanStepPtr projectOnlyUsedColumns( const SharedHeader & stream_header, const ColumnIdentifiers & used_column_identifiers); +/** Build converting actions from `source_columns` to `result_columns`, matching by name when the names make that + * unambiguous and falling back to matching by position otherwise. + * + * Name matching is used only when both sides hold the same set of names with no duplicates, which makes the rename a + * pure reordering. Anything else -- a differing count, a repeated name, a source column the result does not mention -- + * goes to position matching, because the caller may be renaming columns rather than reordering them. The position + * branch is load-bearing, not a safety net: a source header can legitimately carry names the result never uses, such + * as the synthetic aggregate-state column behind an optimized trivial `count()`. + * + * `location` names the call site in the trace log emitted when the helper falls back to position. + */ +ActionsDAG makeConvertingActionsPreferNameThenPosition( + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & result_columns, + const ContextPtr & context, + std::string_view location, + bool ignore_constant_values, + bool add_cast_columns, + NameToNameMap * new_names = nullptr); + } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 1fbcff907866..db1e3a5c1ab4 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -942,7 +943,16 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ { /// Source tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertAndFilterSourceStream(*common_header, modified_query_info, nested_storage_snapshot, aliases, row_policy_data_opt, context, child, is_smallest_column_requested); + convertAndFilterSourceStream( + *common_header, + modified_query_info, + nested_storage_snapshot, + merge_storage_snapshot->metadata->getColumns(), + aliases, + row_policy_data_opt, + context, + child, + is_smallest_column_requested); for (const auto & filter_info : pushed_down_filters) { @@ -1627,6 +1637,7 @@ void ReadFromMerge::convertAndFilterSourceStream( const Block & header, SelectQueryInfo & modified_query_info, const StorageSnapshotPtr & snapshot, + const ColumnsDescription & merge_columns, const Aliases & aliases, const RowPolicyDataOpt & row_policy_data_opt, ContextPtr local_context, @@ -1637,12 +1648,117 @@ void ReadFromMerge::convertAndFilterSourceStream( auto pipe_columns = before_block_header->getNamesAndTypesList(); + /// TODO(storage-merge-alias): the analyzer branch below recomputes by hand what the analyzer's own column-alias + /// resolution would produce if it ran end to end on the child plan. It exists because the work is split in two: + /// + /// Step 1, in `getModifiedQueryInfo`: rewrite the query going to the child storage. `replaceColumns` swaps every + /// reference to a Merge-level ALIAS column for its resolved expression, so the child storage is asked for the + /// PHYSICAL columns those expressions need and never sees the alias names. + /// + /// Step 2, here: recompute the alias VALUES at the Merge level from those physical columns, by building a fresh + /// `ActionsDAG`, running `QueryAnalysisPass` over each alias expression and visiting it with + /// `PlannerActionsVisitor`. Each alias output is emitted under its analyzer identifier so the Merge target + /// header, which also uses analyzer identifiers, can pick it up by name. + /// + /// The awkward part is that alias values are computed AFTER the child's read, not inside it. A predicate over an + /// ALIAS column can still use the underlying physical column for index analysis, but only because Step 1 happens + /// to inline the alias expression into the predicate as well, so `KeyCondition` sees `col * 2 > 10` rather than + /// `alias > 10`. Output-side aliases are recomputed here even when the child already produced the same value -- + /// a `Distributed` child, for instance, evaluates alias expressions on the shard and returns them as + /// expression-named columns, which this then computes a second time. + /// + /// The unification would be to have Step 1 emit `__aliasMarker(, '')` instead + /// of a bare resolved expression. `PlannerActionsVisitor` resolves the marker at plan-build time and the call + /// disappears from the resulting `ActionsDAG`, leaving an ordinary action node that computes `` + /// under ``. Predicate and `KeyCondition` analysis would be unaffected, because the marker is a + /// planner-time naming device rather than a runtime expression. Step 2 would then be deletable outright: + /// `pipe_columns` would already carry the alias values under the right names. + /// + /// Left as future work. What is here is correct -- Step 1 and Step 2 together produce the right values -- just not + /// minimal. if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { + /// The Merge table wants its columns under analyzer identifiers (`__table1.a`, ``__table1.`n.a` ``), while alias + /// expressions and `alias.name` speak in plain logical names (`a`, `n.a`). + /// + /// The planner's `TableExpressionData` for the Merge node is not populated yet at this point -- column + /// collection happens later, in `CollectTableExpressionData` -- so the mapping cannot be looked up through + /// `PlannerContext`. Build it here instead, by matching each Merge-declared column name against the suffixes of + /// the identifier names in `header` / `pipe_columns`: a column named `.` or ``.`` `` + /// belongs to the Merge column `C`. + /// + /// Driving the match from the declared Merge schema rather than from a pattern over the analyzer's naming + /// convention is what makes dotted names work, `Nested` subcolumns and backtick-quoted names included. + + /// `getAll` builds a fresh list on every call, so take it once rather than per candidate column. + const auto all_merge_columns = merge_columns.getAll(); + + auto build_plain_to_identifier = [&merge_columns, &all_merge_columns](const auto & candidate_names) + { + std::unordered_map plain_to_identifier; + std::unordered_set ambiguous; + for (const auto & column : candidate_names) + { + /// An exact match means the column carries no analyzer prefix at all. + if (merge_columns.has(column.name)) + { + if (!plain_to_identifier.emplace(column.name, column.name).second) + ambiguous.insert(column.name); + continue; + } + + /// Otherwise look for `.` or ``.`` `` where C is a declared Merge column. Columns + /// matching no declared name are skipped; they are intermediate outputs of the child plan. Branches + /// differ in how they quote a dotted name -- some emit ``__tableN.`n.a` ``, others `__tableN.n.a` -- so + /// accept both. + for (const auto & merge_column : all_merge_columns) + { + const bool dotted = merge_column.name.find('.') != String::npos; + const String want_raw = "." + merge_column.name; + const String want_quoted = dotted ? ("." + backQuote(merge_column.name)) : want_raw; + if (column.name.ends_with(want_quoted) || (dotted && column.name.ends_with(want_raw))) + { + if (!plain_to_identifier.emplace(merge_column.name, column.name).second) + ambiguous.insert(merge_column.name); + break; + } + } + } + for (const auto & ambiguous_name : ambiguous) + plain_to_identifier.erase(ambiguous_name); + return plain_to_identifier; + }; + + const auto header_plain_to_identifier = build_plain_to_identifier(header); + const auto pipe_plain_to_identifier = build_plain_to_identifier(pipe_columns); + for (const auto & alias : aliases) { ActionsDAG actions_dag(pipe_columns); + /// Alias expressions name their inputs in plain logical names, but the child stream's inputs carry analyzer + /// identifiers. Expose every mapped column under its plain name as well, so that `buildQueryTree` below + /// resolves a reference like `a` or `n.a` against an input named `__table1.a` or ``__table1.`n.a` ``. An + /// alias defined over another alias needs this. + for (const auto & [plain, identifier] : pipe_plain_to_identifier) + { + if (plain == identifier) + continue; + + const ActionsDAG::Node * input_node = nullptr; + for (const auto * candidate : actions_dag.getInputs()) + { + if (candidate->result_name == identifier) + { + input_node = candidate; + break; + } + } + + if (input_node) + actions_dag.addAlias(*input_node, plain); + } + QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context); query_tree->setAlias(alias.name); @@ -1653,13 +1769,18 @@ void ReadFromMerge::convertAndFilterSourceStream( collectSets(query_tree, *modified_query_info.planner_context); ColumnNodePtrWithHashSet empty_correlated_columns_set; - PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, false /*use_column_identifier_as_action_node_name*/); + PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, true /*use_column_identifier_as_action_node_name*/); const auto & [nodes, _] = actions_visitor.visit(actions_dag, query_tree); if (nodes.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size()); - actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), alias.name)); + /// Emit the alias output under its analyzer identifier, so the `addMissingDefaults` step further down + /// matches it by name. Emitted under the plain name it would miss, and the expected + /// ``__tableN.`alias.name` `` column would be filled with type defaults instead. + const auto identifier_it = header_plain_to_identifier.find(alias.name); + const String & output_name = identifier_it != header_plain_to_identifier.end() ? identifier_it->second : alias.name; + actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), output_name)); auto expression_step = std::make_unique(child.plan.getCurrentHeader(), std::move(actions_dag)); child.plan.addStep(std::move(expression_step)); } @@ -1733,11 +1854,18 @@ void ReadFromMerge::convertAndFilterSourceStream( } } - auto convert_actions_dag = ActionsDAG::makeConvertingActions( + /// Prefer matching by name. A `Distributed` child inlines ALIAS expressions on the shard and returns them in + /// whatever order the shard's `ActionsDAG` produced, so position matching here can pair an alias column with the + /// wrong source and silently swap values. Position still handles the cases name matching cannot, such as a virtual + /// column that the child names differently. + auto convert_actions_dag = makeConvertingActionsPreferNameThenPosition( current_step_columns, converted_columns, - ActionsDAG::MatchColumnsMode::Position, - local_context); + local_context, + "StorageMerge", + false /*ignore_constant_values*/, + false /*add_cast_columns*/, + nullptr /*new_names*/); auto expression_step = std::make_unique(child.plan.getCurrentHeader(), std::move(convert_actions_dag)); child.plan.addStep(std::move(expression_step)); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 99d815791fea..34efb3fd5e7e 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -306,6 +306,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Block & header, SelectQueryInfo & modified_query_info, const StorageSnapshotPtr & snapshot, + const ColumnsDescription & merge_columns, const Aliases & aliases, const RowPolicyDataOpt & row_policy_data_opt, ContextPtr context, From 714a78b36af9f433e7a68a4c5c72547b5378f47a Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 15 Sep 2026 16:46:52 +0200 Subject: [PATCH 04/18] Materialize __aliasMarker ids after the shard tree is renumbered Reading an `ALIAS` column of a Hybrid table from inside a subquery fails on the initiator: Cannot find column `max(__table1.computed)` in source stream, there are only columns: [max(__table2.computed)] The marker id was built when the marker was injected, from the table alias the initiator had assigned at that moment. `buildQueryTreeForShard` then runs `createUniqueAliasesIfNecessary`, which renumbers the `__tableN` aliases of the shard tree from 1. The header the initiator matches against is built from the renumbered tree, but the id inside the marker is a `String` constant and keeps the old number. The shard duly emits a column under the stale name and nothing matches it. Keep the id as a live `ColumnNode` from injection onward, so the analyzer passes transform it along with the rest of the tree, and materialize it into a `String` constant only at a serialization boundary, in `finalizeAliasMarkersForDistributedSerialization`. There are two such boundaries: `buildQueryTreeDistributed`, right after `buildQueryTreeForShard`, and `executeSubqueryNode`, where a `GLOBAL IN` / `GLOBAL JOIN` subquery is materialized and shipped as a temporary table. Finalizing in the second place also stabilizes the tree hash that names the temporary table, which otherwise hashes a `ColumnNode` whose identifier can still change. The finalize pass walks bottom-up, so a nested marker chain materializes from the inside out, and it does not descend into lambda bodies: a marker written by hand as `arrayMap(x -> __aliasMarker(x, x), ...)` resolves to a lambda parameter that has no table source to build an id from. `FunctionAliasMarker` accepts any second argument now, because for most of its life that argument is a `ColumnNode` rather than a `String`. The rejection it replaced bought nothing: the function is a pass-through identity, so a marker a user writes by hand is harmless, and turning it into a server-side error only made valid SQL fail. For the same reason the id argument is no longer declared always-constant. Both planner sites fall back to naming the node after the expression the marker wraps when no finalized id is present, which is the name it would have had with no marker at all. Two deliberate differences from the original change in https://github.com/Altinity/ClickHouse/pull/1844. `NormalizeAliasMarkerVisitor` is kept: it flattens nested marker chains at SQL-render time, which happens after finalization, and the nested-chain tests pass with it in place. And the finalize pass never throws: a marker it cannot build an id for is left alone rather than raising a `LOGICAL_ERROR` that valid user SQL can reach. Verified against the 20 staged regression tests from https://github.com/Altinity/ClickHouse/pull/1844: `03842_hybrid_alias_issue_1424` and `03933_alias_marker_direct_use_no_logical_error` now pass, nothing regressed, and the two remaining failures are tests asserting a `serverError` that upstream fixes no longer raise. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- src/Analyzer/Utils.cpp | 58 +++++++++++++++++++++++++ src/Analyzer/Utils.h | 13 ++++++ src/Functions/identity.h | 36 +++++++++++++-- src/Planner/PlannerActionsVisitor.cpp | 42 +++++++++++------- src/Storages/StorageDistributed.cpp | 8 +++- src/Storages/buildQueryTreeForShard.cpp | 50 +++++++++++++-------- 6 files changed, 169 insertions(+), 38 deletions(-) diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index fe5ddbdd6d7f..754cd584aedc 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -1017,6 +1017,64 @@ void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const Stri function_node.resolveAsAggregateFunction(std::move(aggregate_function)); } +namespace +{ + +class FinalizeAliasMarkersVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit FinalizeAliasMarkersVisitor(ContextPtr context_) : context(std::move(context_)) {} + + /// Visit children first, so a nested marker chain is materialized from the inside out. + bool shouldTraverseTopToBottom() const { return false; } + + static bool needChildVisit(const QueryTreeNodePtr & parent, const QueryTreeNodePtr &) + { + /// Do not descend into a lambda body. A marker there -- `arrayMap(x -> __aliasMarker(x, x), ...)` -- is a + /// per-row identity the user wrote, not a transport marker, and its column argument resolves to the lambda + /// parameter, which has no table source to build an id from. + return !parent || parent->getNodeType() != QueryTreeNodeType::LAMBDA; + } + + void visitImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "__aliasMarker") + return; + + auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() != 2 || !arguments[0] || !arguments[1]) + return; + + /// Already materialized on an earlier hop. + if (const auto * id_node = arguments[1]->as(); id_node && isString(id_node->getResultType())) + return; + + const auto * column_node = arguments[1]->as(); + if (!column_node) + return; + + const auto & column_source = column_node->getColumnSourceOrNull(); + if (!column_source || !column_source->hasAlias()) + return; + + auto alias_id = column_source->getAlias() + "." + column_node->getColumnName(); + arguments[1] = std::make_shared(std::move(alias_id), std::make_shared()); + resolveOrdinaryFunctionNodeByName(*function_node, "__aliasMarker", context); + } + +private: + ContextPtr context; +}; + +} + +void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, const ContextPtr & context) +{ + FinalizeAliasMarkersVisitor visitor(context); + visitor.visit(node); +} + std::pair getExpressionSource(const QueryTreeNodePtr & node) { if (const auto * column = node->as()) diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 1e1182c76649..08cf5215426a 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -157,6 +157,19 @@ void resolveOrdinaryFunctionNodeByName(FunctionNode & function_node, const Strin /// Arguments and parameters are taken from the node. void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const String & function_name); +/** Materialize the id argument of every `__aliasMarker` in the tree, turning the `ColumnNode` it carries into the + * `String` constant the shard will read. + * + * Call this immediately before the tree is rendered as SQL for a shard, and no earlier. The id is the column's + * analyzer identifier, and `createUniqueAliasesIfNecessary` -- which runs inside `buildQueryTreeForShard` -- is what + * settles the `__tableN` part of it. An id frozen before that point names a table alias that no longer exists by the + * time the shard header is built. + * + * A marker whose id is already a `String` constant was materialized on an earlier hop and is left alone. A marker + * carrying anything else is a user-written one, not ours, and is also left alone. + */ +void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, const ContextPtr & context); + /// Returns single source of expression node. /// First element of pair is source node, can be nullptr if there are no sources or multiple sources. /// Second element of pair is true if there is at most one source, false if there are multiple sources. diff --git a/src/Functions/identity.h b/src/Functions/identity.h index 9c2ae607de1d..8278fff52af7 100644 --- a/src/Functions/identity.h +++ b/src/Functions/identity.h @@ -108,6 +108,35 @@ struct AliasMarkerName static constexpr auto name = "__aliasMarker"; }; +/** `__aliasMarker(expr, id)` is an internal pass-through identity. It returns `expr` untouched; `id` exists only to + * give the expression a stable name in the planner's `ActionsDAG`. + * + * It is injected when an `ALIAS` column is inlined into its defining expression for transport to a shard. The + * initiator still sees the un-inlined column, so without the marker the shard would name the output after the + * expression (`multiply(__table1.value, 2)`) while the initiator expects the column (`__table1.computed`), and the + * two headers could not be matched by name. + * + * The marker carries the low-level column identity, not the user's SQL alias. A SQL alias cannot do this job: it + * participates in user-visible query semantics, it can collide with names the user chose, and in the + * mergeable-state path the projection step that would normally apply it is skipped. + * + * This is also why the marker is not `__actionName`. `__actionName` survives into the `ActionsDAG` as a function node + * with a forced result name; `__aliasMarker` is consumed into an alias on top of its child, which is what keeps the + * expression behaving like a distinct logical column. + * + * The second argument travels in two forms. Between injection and serialization it is a `ColumnNode`, so the ordinary + * analyzer passes keep transforming it along with everything else -- in particular `createUniqueAliasesIfNecessary`, + * which assigns the final `__tableN` aliases. `finalizeAliasMarkersForDistributedSerialization` then materializes it + * into a `String` constant just before the query is rendered as SQL. Freezing the id any earlier captures a table + * alias that later changes, and the shard then emits a name the initiator never asked for. + * + * Both forms are accepted here, and so is anything else: a user can write `__aliasMarker(x, x)` and it behaves as an + * identity. Rejecting unexpected arguments in the function would turn user SQL into a server-side error for no gain, + * since the marker is harmless by construction. + * + * This is a bridge for as long as distributed transport still goes through SQL text. Once query plan serialization + * replaces that boundary, the marker should become unnecessary. + */ class FunctionAliasMarker : public IFunction { public: @@ -116,7 +145,9 @@ class FunctionAliasMarker : public IFunction String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + /// The id argument is a constant only after finalization. Before that it is a `ColumnNode`, and between the two + /// the function must still resolve. + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; } bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -125,9 +156,6 @@ class FunctionAliasMarker : public IFunction if (arguments.size() != 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - if (!WhichDataType(arguments[1]).isString()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); - return arguments.front(); } diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index cf347f8df6c4..147dedce43e1 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -95,6 +95,16 @@ String calculateActionNodeNameWithCastIfNeeded(const ConstantNode & constant_nod return buffer.str(); } +/// Return a `__aliasMarker`'s finalized id, or an empty string when it has none yet. Between injection and +/// serialization the id is a `ColumnNode`, and a user-written marker can hold anything at all. +String tryExtractAliasMarkerId(const QueryTreeNodePtr & id_argument) +{ + if (const auto * id_node = id_argument->as(); id_node && isString(id_node->getResultType())) + return id_node->getValue().safeGet(); + + return {}; +} + class ActionNodeNameHelper { public: @@ -198,18 +208,19 @@ class ActionNodeNameHelper const auto & function_node = node->as(); if (function_node.getFunctionName() == "__aliasMarker") { - /// Perform sanity check, because user may call this function with unexpected arguments const auto & function_argument_nodes = function_node.getArguments().getNodes(); - if (function_argument_nodes.size() == 2) - { - if (const auto * second_argument = function_argument_nodes.at(1)->as()) - { - if (isString(second_argument->getResultType())) - result = second_argument->getValue().safeGet(); - } - } + if (function_argument_nodes.size() != 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - /// Empty node name is not allowed and leads to logical errors + result = tryExtractAliasMarkerId(function_argument_nodes.at(1)); + + /// No finalized id: either the marker has not reached its serialization boundary yet, or a user + /// wrote one by hand. Name the node after the expression it wraps, which is what the name would + /// have been without a marker at all. + if (result.empty()) + result = calculateActionNodeName(function_argument_nodes.at(0)); + + /// An empty node name is not allowed and leads to logical errors. if (result.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); break; @@ -1259,15 +1270,14 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi if (function_arguments.size() != 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - const auto * alias_id_node = function_arguments.at(1)->as(); - if (!alias_id_node || !isString(alias_id_node->getResultType())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); + auto [child_name, levels] = visitImpl(function_arguments.at(0)); - const auto & alias_id = alias_id_node->getValue().safeGet(); + /// Without a finalized id the marker adds no identity of its own, so it resolves to its payload. That happens + /// for a marker the user wrote by hand, and for one that has not reached a serialization boundary yet. + auto alias_id = tryExtractAliasMarkerId(function_arguments.at(1)); if (alias_id.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); + alias_id = child_name; - auto [child_name, levels] = visitImpl(function_arguments.at(0)); if (alias_id == child_name) return {child_name, levels}; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index fe73ea18b98d..e5b6a08daa86 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1103,8 +1103,14 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, rewriteJoinToGlobalJoinIfNeeded(query_node.getJoinTree()); } - return buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify, /*allow_global_join_for_right_table*/ false); + auto shard_query_tree = buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify, /*allow_global_join_for_right_table*/ false); + /// `buildQueryTreeForShard` runs `createUniqueAliasesIfNecessary`, which is what settles the `__tableN` aliases the + /// marker ids are built from. Materialize the ids only now, so each one names the table alias the shard will + /// actually use rather than the one the initiator happened to assign before the renumbering. + finalizeAliasMarkersForDistributedSerialization(shard_query_tree, query_context); + + return shard_query_tree; } std::optional> tryGetParamTypeAndName(const ASTPtr & node) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index eb1374aad67d..392d3124af5b 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -98,36 +98,31 @@ QueryTreeNodePtr getInlineableAliasColumnExpression(const QueryTreeNodePtr & nod return column_node->getExpression()->clone(); } -/// The id a marker carries for `column_node`. The source's alias qualifies the column name whenever the source has one, -/// so two same-named `ALIAS` columns coming from different tables stay distinguishable in the shard header. -String aliasMarkerIdFor(const ColumnNode & column_node) -{ - const auto & column_source = column_node.getColumnSourceOrNull(); - const auto & source_alias = column_source ? column_source->getAlias() : String{}; - if (source_alias.empty()) - return column_node.getColumnName(); - return source_alias + "." + column_node.getColumnName(); -} - /// Wrap an inlined `ALIAS` expression in `__aliasMarker` so it carries the identity of the column it came from. -/// An expression that is already a marker gets its id refreshed rather than a second wrapper around it. +/// +/// The id goes in as a live `ColumnNode`, not as a finished string. Later analyzer passes still have to run -- +/// `createUniqueAliasesIfNecessary` in particular, which assigns the `__tableN` aliases the id is built from -- and a +/// `ColumnNode` is transformed by those passes exactly as the rest of the tree is. +/// `finalizeAliasMarkersForDistributedSerialization` turns it into a `String` constant at the serialization boundary. +/// +/// An expression that is already a marker has its id replaced rather than gaining a second wrapper. QueryTreeNodePtr wrapInAliasMarker(QueryTreeNodePtr expression, const ColumnNode & column_node, const ContextPtr & context) { - auto alias_id = aliasMarkerIdFor(column_node); + auto marker_id = std::make_shared(column_node.getColumn(), column_node.getColumnSourceOrNull()); if (auto * function_node = expression->as(); function_node && function_node->getFunctionName() == "__aliasMarker") { auto & marker_arguments = function_node->getArguments().getNodes(); if (marker_arguments.size() == 2) - marker_arguments[1] = std::make_shared(std::move(alias_id), std::make_shared()); + marker_arguments[1] = std::move(marker_id); return expression; } QueryTreeNodes arguments; arguments.reserve(2); arguments.emplace_back(std::move(expression)); - arguments.emplace_back(std::make_shared(std::move(alias_id), std::make_shared())); + arguments.emplace_back(std::move(marker_id)); auto marker_node = std::make_shared("__aliasMarker"); marker_node->getArguments().getNodes() = std::move(arguments); @@ -175,6 +170,20 @@ struct AliasColumnInliner { } + if (auto * marker_node = node->as(); + marker_node && marker_node->getFunctionName() == "__aliasMarker") + { + auto & marker_arguments = marker_node->getArguments().getNodes(); + if (marker_arguments.size() == 2) + { + /// Descend into the payload only. The second argument is the column reference the marker exists to + /// record, and inlining it would expand the very `ALIAS` column whose identity it carries. It has to + /// reach `finalizeAliasMarkersForDistributedSerialization` as a `ColumnNode`. + inlineExpression(marker_arguments[0]); + return; + } + } + auto * join_node = node->as(); const bool using_join = join_node && join_node->isUsingJoinExpression(); @@ -633,7 +642,14 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, ContextMutablePtr & mutable_context, size_t subquery_depth) { - const auto subquery_hash = subquery_node->getTreeHash(); + /// A `GLOBAL IN` / `GLOBAL JOIN` subquery is materialized here and shipped as a temporary table, so this is a + /// serialization boundary too and the marker ids have to be finalized before the subquery runs. Finalizing also + /// makes the tree hash below stable: an unmaterialized marker hashes its `ColumnNode`, whose identifier can still + /// change, which would key the same subquery under two different temporary table names. + auto subquery_node_to_execute = subquery_node->clone(); + finalizeAliasMarkersForDistributedSerialization(subquery_node_to_execute, mutable_context); + + const auto subquery_hash = subquery_node_to_execute->getTreeHash(); const auto temporary_table_name = fmt::format("_data_{}", toString(subquery_hash)); const auto & external_tables = mutable_context->getExternalTables(); @@ -651,7 +667,7 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, auto context_copy = Context::createCopy(mutable_context); updateContextForSubqueryExecution(context_copy); - InterpreterSelectQueryAnalyzer interpreter(subquery_node, context_copy, subquery_options); + InterpreterSelectQueryAnalyzer interpreter(subquery_node_to_execute, context_copy, subquery_options); auto & query_plan = interpreter.getQueryPlan(); auto sample_block_with_unique_names = *query_plan.getCurrentHeader(); From 629e2c730769070729e3a6a2c05d02d513b1b197 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 15 Sep 2026 16:55:07 +0200 Subject: [PATCH 05/18] tests: assert results where the expected errors no longer happen Three of the staged regression tests expect a `serverError` that upstream fixes stopped raising, so each was asserting a bug that no longer exists. Replace the assertion with the result the query now returns, and fix the framing that went with it. `03934` claimed `enable_alias_marker` was a correctness toggle: with the marker off, a distributed-over-distributed read of a `String` ALIAS and a `UInt64` ALIAS swapped the two columns and failed with `CANNOT_PARSE_TEXT`. Upstream fixed the column ordering, so the setting is what it was always meant to be -- a way for an initiator to stop emitting `__aliasMarker` for a mixed-version cluster whose shards do not understand it. The test now runs the same query with the marker on and off and requires both to return the same rows. `05053` expected `NUMBER_OF_COLUMNS_DOESNT_MATCH` for two ALIAS columns over the same expression with the marker off. `buildShardCollapseFanOut` from https://github.com/ClickHouse/ClickHouse/pull/107913 reconstructs the column the shard deduplicated away whether or not a marker is present, so that variant returns the same single row as the other six. `05060` expected `UNKNOWN_IDENTIFIER` from a Hybrid/MergeTree/Iceberg join under `object_storage_cluster_join_mode = 'local'`. It now returns the same rows as the `'allow'` variant beside it, and is held to them. Each reference was taken from a sibling query in the same test that must produce identical output, so a wrong reference here would mean the test was already wrong. `05060` also gained a header note: it needs the stateless S3 mock on localhost:11111, it needs a server listening on `127.0.0.3` for `test_cluster_one_shard_three_replicas_localhost`, and it needs `enable_parallel_blocks_marshalling = 0` until the `DISTINCT` over `ColumnBLOB` abort is fixed. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- ...uted_alias_marker_setting_effect.reference | 15 ++++++++++++++- ...istributed_alias_marker_setting_effect.sql | 19 +++++++++++-------- ...istributed_alias_same_expression.reference | 1 + ...5053_distributed_alias_same_expression.sql | 13 +++++++++++-- ...ybrid_unknown_table_exact_schema.reference | 5 +++++ ...5060_hybrid_unknown_table_exact_schema.sql | 14 ++++++++++++-- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference index cc0e49a8fdf9..d015dccfb64f 100644 --- a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference @@ -12,4 +12,17 @@ x a_str inner_c 10 aaaa 11 10 aaaa 11 10 aaaa 11 -marker_off_reintroduces_swap +marker_off +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql index 4100828c9eb5..648c98d7553b 100644 --- a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql @@ -1,8 +1,11 @@ --- Demonstrates that enable_alias_marker is a correctness toggle for distributed ALIAS columns. --- Distributed-over-distributed with a String ALIAS (`a_str`) and a UInt64 ALIAS (`inner_c`): --- * marker ON -> columns reconciled by name, correct results. --- * marker OFF -> the inlined ALIAS expansion swaps columns; the String 'aaaa' is routed into --- the UInt64 `inner_c` slot and the query fails with CANNOT_PARSE_TEXT. +-- `enable_alias_marker` must not change results. It exists so an initiator can stop emitting +-- `__aliasMarker` for a mixed-version cluster whose shards do not understand it, which is a +-- transport concern, not a correctness one. +-- +-- Distributed-over-distributed with a String ALIAS (`a_str`) and a UInt64 ALIAS (`inner_c`). This +-- shape used to swap the two columns with the marker off, routing the String 'aaaa' into the +-- UInt64 `inner_c` slot and failing with CANNOT_PARSE_TEXT. Upstream fixed the underlying column +-- ordering, so both settings now return the same rows, and this test holds them to that. DROP TABLE IF EXISTS t_se_local; DROP TABLE IF EXISTS t_se_inner; DROP TABLE IF EXISTS t_se_outer; @@ -27,12 +30,12 @@ ORDER BY x SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0 FORMAT TSVWithNames; -SELECT 'marker_off_reintroduces_swap'; --- No output format header here: the query errors mid-execution, so it must not stream a header. +SELECT 'marker_off'; SELECT x, a_str, inner_c FROM t_se_outer ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 0, prefer_localhost_replica = 0, serialize_query_plan = 0; -- { serverError CANNOT_PARSE_TEXT } +SETTINGS enable_analyzer = 1, enable_alias_marker = 0, prefer_localhost_replica = 0, serialize_query_plan = 0 +FORMAT TSVWithNames; DROP TABLE t_se_outer; DROP TABLE t_se_inner; diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference index 279f11b9bf19..8ddb26d0b961 100644 --- a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference @@ -5,6 +5,7 @@ second third 1999-03-29 01:15:33.000 fourth +1999-03-29 01:15:33.000 fifth 1999-03-29 01:15:33.000 sixth diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql index 8aa596998571..20cf77930d0b 100644 --- a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql @@ -1,5 +1,14 @@ --- Regression coverage for distributed ORDER BY + ALIAS columns with identical expressions. +-- Two ALIAS columns over the same expression, read through `remote` with ORDER BY. The shard's +-- ActionsDAG deduplicates the two identical expressions into one output column, so its header is a +-- column short of what the initiator expects. +-- +-- Every variant below must return the same single row. `enable_alias_marker = 0` is covered too: +-- the marker keeps the two columns distinct in transport, but it is not what makes this shape work. +-- `buildShardCollapseFanOut` reconstructs the missing column either way, so turning the marker off +-- must not change the answer. +-- -- Related issue: https://github.com/ClickHouse/ClickHouse/issues/79916 +-- Fixed upstream by: https://github.com/ClickHouse/ClickHouse/pull/107913 DROP TABLE IF EXISTS test_alias_same_expr_remote; @@ -39,7 +48,7 @@ SELECT dt, alias_String_7_0, alias_String_7_1 FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) ORDER BY dt LIMIT 1 -SETTINGS enable_analyzer = 1, enable_alias_marker = 0; -- { serverError NUMBER_OF_COLUMNS_DOESNT_MATCH } +SETTINGS enable_analyzer = 1, enable_alias_marker = 0; SELECT 'fifth'; SELECT dt, alias_String_7_0, alias_String_7_1 diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference index dcebc31194b1..286fa88dd6a4 100644 --- a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference @@ -22,6 +22,11 @@ Louis 1519 1519 Oliver 1654 1654 William 9044 9044 issue_1208_join_hybrid_mt_iceberg_local +Frank 8751 8751 8751 +Isaac 3611 3611 3611 +Louis 1519 1519 1519 +Oliver 1654 1654 1654 +William 9044 9044 9044 issue_1208_join_hybrid_mt_iceberg_allow Frank 8751 8751 8751 Isaac 3611 3611 3611 diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql index a37c991684b4..55d247058b3f 100644 --- a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql @@ -1,3 +1,13 @@ +-- Hybrid table joined against a MergeTree and an Iceberg table, covering Altinity#1208, #1209 and +-- #1422. Both `object_storage_cluster_join_mode` values must return the same rows: `'local'` used +-- to raise UNKNOWN_IDENTIFIER here, and no longer does, so it is held to the `'allow'` result. +-- +-- Needs the stateless S3 mock on localhost:11111, and a server listening on 127.0.0.3 as well as +-- 127.0.0.1 and 127.0.0.2, because `icebergCluster` below reads through +-- `test_cluster_one_shard_three_replicas_localhost`. Without the third address the query still +-- returns the right rows by failing over, but the connection warnings on stderr fail the test. +-- Run with `enable_parallel_blocks_marshalling = 0` until the DISTINCT-over-ColumnBLOB abort is +-- fixed. SET allow_experimental_hybrid_table = 1, enable_analyzer = 1, prefer_localhost_replica = 0, @@ -201,7 +211,7 @@ FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h. FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col ORDER BY h.string_col LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'local'; -- { serverError UNKNOWN_IDENTIFIER } +SETTINGS object_storage_cluster_join_mode = 'local'; SELECT 'issue_1208_join_hybrid_mt_iceberg_allow'; SELECT @@ -247,7 +257,7 @@ FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h. FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col LIMIT 10 SETTINGS object_storage_cluster_join_mode = 'local' -FORMAT Null; -- { serverError UNKNOWN_IDENTIFIER } +FORMAT Null; SELECT h.string_col, From 9c167de0b8d251c1d387d6ff432b6d7f9f49b567 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 19 Sep 2026 09:12:06 +0200 Subject: [PATCH 06/18] Finalize __aliasMarker inside lambdas; drop a no-op converting helper A review of the three preceding commits turned up four things worth changing. `finalizeAliasMarkersForDistributedSerialization` refused to descend into a lambda body, but the inliner has no such limit: `inlineExpression` recurses through every child, so `arrayMap(x -> x + computed, dist)` wrapped `computed` in a marker that nothing then materialized. Its id stayed a `ColumnNode` and was rendered straight back into the shipped SQL as `__table1.computed`, leaving the shard to resolve the very `ALIAS` column the inlining exists to remove. Where the column is declared on the `Distributed` table and not on the local one, the shard cannot. The guard is gone. The case it was written for -- a hand-written `__aliasMarker(x, x)` inside a lambda, whose id is the lambda parameter -- is now handled by checking the id's column source instead. A lambda parameter's source is the `LambdaNode`, which `createUniqueAliasesIfNecessary` never aliases, so it is recognisable without refusing to look. `makeConvertingActionsPreferNameThenPosition` is deleted. Its only call site builds `converted_columns[i]` out of `current_step_columns[i]`, and every branch of that loop either copies the source name through or -- in the one branch reachable only when the source name is absent from `header` -- substitutes a name that cannot then appear anywhere in the result set. So either every name matches its own position, in which case `MatchColumnsMode::Name` and `::Position` pair the same columns, or the two name sets differ and the helper falls back to position anyway. The column swap in `05057` is fixed by emitting alias outputs under analyzer identifiers, not by this. `src/Planner/Utils.{h,cpp}` are back to their previous contents. Finalization moved from `buildQueryTreeDistributed` into the end of `buildQueryTreeForShard`. Four call sites build a shard tree and only one finalized it; nothing is broken today, because only the `Distributed` path injects markers at all, but wiring a second injection site would have shipped markers nobody materializes. Last in the function rather than straight after `createUniqueAliasesIfNecessary`, so a long id still passes `ReplaceLongConstWithScalarVisitor` as a `ColumnNode` rather than as a `String` that visitor could replace with a scalar. The plain-name to analyzer-identifier mapping in `ReadFromMerge::convertAndFilterSourceStream` now takes the longest matching declared column name rather than the first. This is not a bug fix: reintroducing the first-match behaviour leaves `05065` passing, because an analyzer identifier backquotes a dotted name and the child stream never emits a dotted identifier. It removes the mapping's dependence on an invariant enforced two layers away. Two review findings were not acted on. An assertion for an unfinalized marker id cannot be written: nothing at the planner can tell an injected marker that missed its boundary from one a user typed, and `03933` runs exactly that shape on purpose. And `NormalizeAliasMarkerVisitor` does not discard nested marker chains -- it never descends into a marker's own arguments and collapses one only when the payload is itself a marker, so the only id it drops names an intermediate action node. Both are recorded as comments where a future reader will look. Upstream https://github.com/ClickHouse/ClickHouse/pull/107700 also calls `inlineAliasColumns` from `ClusterProxy/executeQuery.cpp` and `findParallelReplicasQuery.cpp`, fixing `NO_SUCH_COLUMN_IN_TABLE` for an `ALIAS` column on the shipped side of a parallel-replicas query. That bug is real on this branch's base and predates this work. It is left alone: this `inlineAliasColumns` injects markers when `enable_alias_marker` is on, which is the default, so wiring those sites would put markers on two paths that have never carried one. New tests: `05064_alias_marker_in_lambda_over_distributed` covers the lambda case, and `05065_shadowed_dotted_alias_merge_over_distributed` guards the values for a `Merge` schema declaring both `b` and `` `a.b` ``. Related: https://github.com/Altinity/ClickHouse/pull/1844 Related: https://github.com/ClickHouse/ClickHouse/pull/107700 Co-Authored-By: Claude Opus 5 --- src/Analyzer/Utils.cpp | 17 +++-- src/Planner/PlannerActionsVisitor.cpp | 4 +- src/Planner/Utils.cpp | 61 ----------------- src/Planner/Utils.h | 22 ------ src/Storages/StorageDistributed.cpp | 11 +-- src/Storages/StorageMerge.cpp | 41 ++++++----- src/Storages/buildQueryTreeForShard.cpp | 13 +++- ...arker_in_lambda_over_distributed.reference | 9 +++ ...lias_marker_in_lambda_over_distributed.sql | 43 ++++++++++++ ...ted_alias_merge_over_distributed.reference | 9 +++ ...ed_dotted_alias_merge_over_distributed.sql | 68 +++++++++++++++++++ 11 files changed, 180 insertions(+), 118 deletions(-) create mode 100644 tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference create mode 100644 tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql create mode 100644 tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference create mode 100644 tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index 754cd584aedc..cc9f77498055 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -1028,14 +1028,6 @@ class FinalizeAliasMarkersVisitor : public InDepthQueryTreeVisitor __aliasMarker(x, x), ...)` -- is a - /// per-row identity the user wrote, not a transport marker, and its column argument resolves to the lambda - /// parameter, which has no table source to build an id from. - return !parent || parent->getNodeType() != QueryTreeNodeType::LAMBDA; - } - void visitImpl(QueryTreeNodePtr & node) { auto * function_node = node->as(); @@ -1055,7 +1047,14 @@ class FinalizeAliasMarkersVisitor : public InDepthQueryTreeVisitorgetColumnSourceOrNull(); - if (!column_source || !column_source->hasAlias()) + if (!column_source) + return; + + /// A lambda parameter -- `arrayMap(x -> __aliasMarker(x, x), ...)` written by hand -- has the `LambdaNode` as + /// its source. There is no table alias to build an id from, and the marker is a per-row identity rather than a + /// transport marker, so leave it as it is. A marker this pass injected inside a lambda body does not land here: + /// its id names an `ALIAS` column of a table, and the table expression is its source. + if (column_source->getNodeType() == QueryTreeNodeType::LAMBDA || !column_source->hasAlias()) return; auto alias_id = column_source->getAlias() + "." + column_node->getColumnName(); diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 147dedce43e1..3b2fb3c78fa9 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -216,7 +216,9 @@ class ActionNodeNameHelper /// No finalized id: either the marker has not reached its serialization boundary yet, or a user /// wrote one by hand. Name the node after the expression it wraps, which is what the name would - /// have been without a marker at all. + /// have been without a marker at all. The two cases are not distinguishable here -- a hand-written + /// `__aliasMarker(x, x)` looks exactly like an injected one whose id is still a `ColumnNode` -- so + /// this cannot be turned into an assertion without rejecting valid SQL (03933). if (result.empty()) result = calculateActionNodeName(function_argument_nodes.at(0)); diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index e375619925e2..f48515204da2 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -1,8 +1,5 @@ #include -#include -#include - #include #include #include @@ -850,62 +847,4 @@ QueryPlanStepPtr projectOnlyUsedColumns( return step; } -namespace -{ - -/// True when both sides hold the same set of names and neither repeats one, so matching by name is a pure reordering. -bool canMatchByNameWithoutAmbiguity(const ColumnsWithTypeAndName & source, const ColumnsWithTypeAndName & result) -{ - if (source.size() != result.size()) - return false; - - NameSet source_names; - NameSet result_names; - - for (const auto & source_column : source) - if (!source_names.insert(source_column.name).second) - return false; - - for (const auto & result_column : result) - if (!result_names.insert(result_column.name).second) - return false; - - return source_names == result_names; -} - -} - -ActionsDAG makeConvertingActionsPreferNameThenPosition( - const ColumnsWithTypeAndName & source_columns, - const ColumnsWithTypeAndName & result_columns, - const ContextPtr & context, - std::string_view location, - bool ignore_constant_values, - bool add_cast_columns, - NameToNameMap * new_names) -{ - const bool match_by_name = canMatchByNameWithoutAmbiguity(source_columns, result_columns); - - if (!match_by_name) - { - /// `LOG_IMPL` checks the level before formatting, so the two `dumpNames` calls stay off the hot path. - static const auto log = getLogger("Planner"); - LOG_TEST( - log, - "Matching by position at {}, because the names do not line up as a set: source=[{}] result=[{}]", - location, - Block(source_columns).dumpNames(), - Block(result_columns).dumpNames()); - } - - return ActionsDAG::makeConvertingActions( - source_columns, - result_columns, - match_by_name ? ActionsDAG::MatchColumnsMode::Name : ActionsDAG::MatchColumnsMode::Position, - context, - ignore_constant_values, - add_cast_columns, - new_names); -} - } diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 10858dc94e3a..97385e9f56fc 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -10,8 +10,6 @@ #include #include - -#include #include #include @@ -126,24 +124,4 @@ QueryPlanStepPtr projectOnlyUsedColumns( const SharedHeader & stream_header, const ColumnIdentifiers & used_column_identifiers); -/** Build converting actions from `source_columns` to `result_columns`, matching by name when the names make that - * unambiguous and falling back to matching by position otherwise. - * - * Name matching is used only when both sides hold the same set of names with no duplicates, which makes the rename a - * pure reordering. Anything else -- a differing count, a repeated name, a source column the result does not mention -- - * goes to position matching, because the caller may be renaming columns rather than reordering them. The position - * branch is load-bearing, not a safety net: a source header can legitimately carry names the result never uses, such - * as the synthetic aggregate-state column behind an optimized trivial `count()`. - * - * `location` names the call site in the trace log emitted when the helper falls back to position. - */ -ActionsDAG makeConvertingActionsPreferNameThenPosition( - const ColumnsWithTypeAndName & source_columns, - const ColumnsWithTypeAndName & result_columns, - const ContextPtr & context, - std::string_view location, - bool ignore_constant_values, - bool add_cast_columns, - NameToNameMap * new_names = nullptr); - } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index e5b6a08daa86..2bae73f364a1 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1103,14 +1103,9 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, rewriteJoinToGlobalJoinIfNeeded(query_node.getJoinTree()); } - auto shard_query_tree = buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify, /*allow_global_join_for_right_table*/ false); - - /// `buildQueryTreeForShard` runs `createUniqueAliasesIfNecessary`, which is what settles the `__tableN` aliases the - /// marker ids are built from. Materialize the ids only now, so each one names the table alias the shard will - /// actually use rather than the one the initiator happened to assign before the renumbering. - finalizeAliasMarkersForDistributedSerialization(shard_query_tree, query_context); - - return shard_query_tree; + /// `buildQueryTreeForShard` materializes the marker ids on the way out, after the `__tableN` renumbering they are + /// built from. + return buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify, /*allow_global_join_for_right_table*/ false); } std::optional> tryGetParamTypeAndName(const ASTPtr & node) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index db1e3a5c1ab4..c606d8cc3e6a 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1711,18 +1711,26 @@ void ReadFromMerge::convertAndFilterSourceStream( /// matching no declared name are skipped; they are intermediate outputs of the child plan. Branches /// differ in how they quote a dotted name -- some emit ``__tableN.`n.a` ``, others `__tableN.n.a` -- so /// accept both. + /// + /// Take the longest matching name rather than the first. A table declaring both `b` and `` `a.b` `` + /// makes `__table1.a.b` end with `.b` as well, so a first match in schema order could hand `b` the + /// wrong column. Nothing reaches that state today -- an analyzer identifier backquotes a dotted name, + /// so the raw form only arrives from a child that never emits a dotted identifier at all -- and the + /// longest match is what keeps the mapping from depending on that. + const String * best_match = nullptr; for (const auto & merge_column : all_merge_columns) { const bool dotted = merge_column.name.find('.') != String::npos; const String want_raw = "." + merge_column.name; const String want_quoted = dotted ? ("." + backQuote(merge_column.name)) : want_raw; - if (column.name.ends_with(want_quoted) || (dotted && column.name.ends_with(want_raw))) - { - if (!plain_to_identifier.emplace(merge_column.name, column.name).second) - ambiguous.insert(merge_column.name); - break; - } + if (!column.name.ends_with(want_quoted) && !(dotted && column.name.ends_with(want_raw))) + continue; + if (!best_match || best_match->size() < merge_column.name.size()) + best_match = &merge_column.name; } + + if (best_match && !plain_to_identifier.emplace(*best_match, column.name).second) + ambiguous.insert(*best_match); } for (const auto & ambiguous_name : ambiguous) plain_to_identifier.erase(ambiguous_name); @@ -1769,6 +1777,12 @@ void ReadFromMerge::convertAndFilterSourceStream( collectSets(query_tree, *modified_query_info.planner_context); ColumnNodePtrWithHashSet empty_correlated_columns_set; + /// `true` not for the naming it implies -- no column identifiers are registered for this table expression, + /// so `calculateActionNodeName` falls back to the plain column name either way -- but for what `false` + /// does to an `ALIAS` column: it replaces the column node with a visit of its defining expression. An + /// `ALIAS` defined over another one (`b ALIAS a + 1` over `a ALIAS x + 1`) would then expand `a` again + /// rather than read the value the child stream already carries, and the physical columns underneath it + /// need not be in that stream at all. The plain-name aliases added above are what the lookup then hits. PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, true /*use_column_identifier_as_action_node_name*/); const auto & [nodes, _] = actions_visitor.visit(actions_dag, query_tree); @@ -1854,18 +1868,13 @@ void ReadFromMerge::convertAndFilterSourceStream( } } - /// Prefer matching by name. A `Distributed` child inlines ALIAS expressions on the shard and returns them in - /// whatever order the shard's `ActionsDAG` produced, so position matching here can pair an alias column with the - /// wrong source and silently swap values. Position still handles the cases name matching cannot, such as a virtual - /// column that the child names differently. - auto convert_actions_dag = makeConvertingActionsPreferNameThenPosition( + /// Position, because the loop above builds `converted_columns[i]` out of `current_step_columns[i]`: the pairing is + /// fixed by that construction rather than by the names, and one of the branches deliberately renames a column. + auto convert_actions_dag = ActionsDAG::makeConvertingActions( current_step_columns, converted_columns, - local_context, - "StorageMerge", - false /*ignore_constant_values*/, - false /*add_cast_columns*/, - nullptr /*new_names*/); + ActionsDAG::MatchColumnsMode::Position, + local_context); auto expression_step = std::make_unique(child.plan.getCurrentHeader(), std::move(convert_actions_dag)); child.plan.addStep(std::move(expression_step)); diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 392d3124af5b..b97c60fcdd1c 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -165,7 +165,12 @@ struct AliasColumnInliner /// An `ALIAS` column may be defined over another one, so keep unwrapping. With markers on the loop stops after /// the first step, because the wrapper is a function node; the column it wrapped is reached through the - /// children below and gets its own marker there, which is what preserves a nested marker chain. + /// children below and gets its own marker there. + /// + /// A marker that ends up nested inside the payload expression -- `computed ALIAS inner * 2` -- is kept as it + /// is. One that ends up as the payload itself -- `computed ALIAS inner` -- is collapsed by + /// `NormalizeAliasMarkerVisitor` when the tree is rendered to SQL, and nothing is lost by that: the inner id + /// names an intermediate action node, while the name the initiator matches against comes from the outer id. while (inlineOnce(node)) { } @@ -1134,6 +1139,12 @@ QueryTreeNodePtr buildQueryTreeForShard( scalar_visitor.visit(query_tree_to_modify); } + /// Last, because `createUniqueAliasesIfNecessary` above is what settles the `__tableN` aliases the marker ids are + /// built from: an id materialized any earlier would name the table alias the initiator happened to assign before + /// the renumbering rather than the one the shard will use. Here rather than in the callers, so that every path + /// that ships a query tree finalizes whatever markers it carries. + finalizeAliasMarkersForDistributedSerialization(query_tree_to_modify, planner_context->getQueryContext()); + return query_tree_to_modify; } diff --git a/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference new file mode 100644 index 000000000000..8262875092b6 --- /dev/null +++ b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference @@ -0,0 +1,9 @@ +control +2 +6 +lambda +[12,22] +[11] +lambda_no_marker +[12,22] +[11] diff --git a/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql new file mode 100644 index 000000000000..848cc84b8f41 --- /dev/null +++ b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql @@ -0,0 +1,43 @@ +-- An `ALIAS` column referenced from inside a lambda body is inlined and marked like any other, but +-- `finalizeAliasMarkersForDistributedSerialization` used to refuse to descend into a lambda. The marker's id then +-- stayed a live `ColumnNode` and was rendered back into the shipped SQL as `__table1.computed`, so the shard had to +-- resolve the very `ALIAS` column the inlining exists to remove. Here the column is declared on the `Distributed` +-- table and not on the local one, so the shard cannot resolve it and the query fails. +-- +-- The finalize pass still leaves a hand-written marker inside a lambda alone, because its id resolves to the lambda +-- parameter rather than to a column of a table -- that is what 03933 covers. + +DROP TABLE IF EXISTS t_local_05064; +DROP TABLE IF EXISTS t_dist_05064; + +CREATE TABLE t_local_05064 (value UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY value; +INSERT INTO t_local_05064 VALUES (1, [10, 20]), (3, [5]); + +CREATE TABLE t_dist_05064 +( + value UInt64, + arr Array(UInt64), + computed UInt64 ALIAS value * 2 +) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05064); + +SELECT 'control'; +SELECT computed +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda'; +SELECT arrayMap(x -> x + computed, arr) +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda_no_marker'; +SELECT arrayMap(x -> x + computed, arr) +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +DROP TABLE t_dist_05064; +DROP TABLE t_local_05064; diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference new file mode 100644 index 000000000000..97dd73615024 --- /dev/null +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference @@ -0,0 +1,9 @@ +local +1 10 100 +2 20 200 +merge_prefer0 +1 10 100 +2 20 200 +merge_prefer1 +1 10 100 +2 20 200 diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql new file mode 100644 index 000000000000..34310f184503 --- /dev/null +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql @@ -0,0 +1,68 @@ +-- End-to-end guard for a `Merge` over `Distributed` whose schema declares both a plain column and a +-- dotted column whose tail is that plain name (`b` and `` `a.b` ``), read as `ALIAS` columns. +-- +-- What this does NOT do: reproduce the first-match bug in the name mapping of +-- `ReadFromMerge::convertAndFilterSourceStream`. That mapping matches declared Merge column names +-- against the suffixes of the child stream's column names, and `__table1.a.b` ends with `.b`, so a +-- first match in schema order would hand `b` the wrong column. No input can put it in that state: +-- the Merge target header carries analyzer identifiers, which backquote a dotted name +-- (`__table1.`a.b``, `PlannerContext.cpp` `buildColumnIdentifier`), and the child stream carries the +-- alias values under the alias name or the expression name, never a dotted analyzer identifier, +-- because the query reaching the child has every Merge-level `ALIAS` reference already replaced by +-- its expression. Verified by reintroducing the first-match behaviour and observing this test still +-- pass. +-- +-- So it asserts the values are right for a schema shape nothing else covers. 05059 covers dotted +-- names on their own. +-- +-- `test_cluster_two_shards` reads the local table twice, so every query groups to dedup. + +DROP TABLE IF EXISTS t_local_05065; +DROP TABLE IF EXISTS t_dist_05065; +DROP TABLE IF EXISTS t_merge_05065; + +CREATE TABLE t_local_05065 +( + id UInt32, + b UInt32 ALIAS id * 10, + `a.b` UInt32 ALIAS id * 100 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO t_local_05065 VALUES (1), (2); + +CREATE TABLE t_dist_05065 AS t_local_05065 +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_local_05065); + +CREATE TABLE t_merge_05065 +( + id UInt32, + b UInt32, + `a.b` UInt32 +) +ENGINE = Merge(currentDatabase(), '^t_dist_05065$'); + +SELECT 'local'; +SELECT id, b, `a.b` +FROM t_local_05065 +GROUP BY id, b, `a.b` +ORDER BY id; + +SELECT 'merge_prefer0'; +SELECT id, b, `a.b` +FROM t_merge_05065 +GROUP BY id, b, `a.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT id, b, `a.b` +FROM t_merge_05065 +GROUP BY id, b, `a.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 1; + +DROP TABLE t_merge_05065; +DROP TABLE t_dist_05065; +DROP TABLE t_local_05065; From a2ea20ffa4a730ed71b6acc4519cc8654cb1a58e Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 19 Sep 2026 09:47:18 +0200 Subject: [PATCH 07/18] Keep the __aliasMarker id out of the shipped GLOBAL JOIN subquery A `GLOBAL JOIN` over a `Distributed` table whose `ALIAS` column is not declared on the local table failed on the shard: Identifier '__table1.foo' cannot be resolved from table with name __table1. In scope SELECT __table1.x AS x, __table1.foo AS foo, __table1.id AS id FROM t_right_local AS __table1 Nothing in that query references `foo` any more -- inlining replaced it with its defining expression. What still names it is the marker's id, which became a live `ColumnNode` so that `createUniqueAliasesIfNecessary` could renumber it. `CollectColumnSourceToColumnsVisitor` records every `ColumnNode` it meets, keyed by source, keeping only the name and type, so the id was gathered as if it were a column the query reads. Those gathered columns become the projection of the subquery `getSubqueryFromTableExpression` builds for the shipped temporary table, rebuilt as bare column nodes with no expression. The alias body is lost on the way, so the shard is asked for a column its storage does not declare. `GLOBAL IN` on a table is unaffected -- it reads the storage's ordinary columns instead of the gathered map. Skip the marker's second argument when collecting. Traversal is top-down, so the marker is always seen before the argument it records. Reachable through `GLOBAL JOIN` and through a `CROSS JOIN` under `find_cross_join`, and through a plain `JOIN` once `prefer_global_in_and_join` promotes it. `05066_global_join_alias_only_on_distributed` covers the alias in the projection and the alias referenced only in a clause, each with the marker on and off, plus the promoted plain `JOIN`. It fails on the previous commit with the error above and passes here. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- src/Storages/buildQueryTreeForShard.cpp | 19 ++++- ...l_join_alias_only_on_distributed.reference | 18 +++++ ..._global_join_alias_only_on_distributed.sql | 75 +++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference create mode 100644 tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index b97c60fcdd1c..0dfdb324db2a 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -317,8 +317,24 @@ class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitoras(); + function_node && function_node->getFunctionName() == "__aliasMarker") + { + const auto & marker_arguments = function_node->getArguments().getNodes(); + if (marker_arguments.size() == 2 && marker_arguments[1]) + marker_id_nodes.insert(marker_arguments[1].get()); + return; + } + auto * column_node = node->as(); - if (!column_node) + if (!column_node || marker_id_nodes.contains(node.get())) return; auto column_source = column_node->getColumnSourceOrNull(); @@ -337,6 +353,7 @@ class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitor column_source_to_columns; + std::unordered_set marker_id_nodes; }; /** Visitor that rewrites IN and JOINs in query and all subqueries according to distributed_product_mode and diff --git a/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference new file mode 100644 index 000000000000..48bb2c730573 --- /dev/null +++ b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference @@ -0,0 +1,18 @@ +projected_marker_on +1 20 +2 40 +3 60 +clause_only_marker_on +2 +3 +projected_marker_off +1 20 +2 40 +3 60 +clause_only_marker_off +2 +3 +implicit_global_join +1 20 +2 40 +3 60 diff --git a/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql new file mode 100644 index 000000000000..c5e86a074721 --- /dev/null +++ b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql @@ -0,0 +1,75 @@ +-- A `GLOBAL JOIN` ships its right side as a temporary table, whose columns are the ones +-- `CollectColumnSourceToColumnsVisitor` gathered from the query tree. The `__aliasMarker` id is a +-- `ColumnNode` naming the `ALIAS` column an inlined expression came from, so it used to be gathered +-- as if it were a column the query reads. The temporary table's subquery is rebuilt from names and +-- types alone, which drops the alias body, and the shard is then asked for a column its local table +-- does not declare. +-- +-- `foo` exists only on the `Distributed` table here, so nothing can resolve it on the shard. + +DROP TABLE IF EXISTS t_left_local_05066; +DROP TABLE IF EXISTS t_left_dist_05066; +DROP TABLE IF EXISTS t_right_local_05066; +DROP TABLE IF EXISTS t_right_dist_05066; + +CREATE TABLE t_left_local_05066 (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_left_local_05066 VALUES (1), (2), (3); + +CREATE TABLE t_left_dist_05066 AS t_left_local_05066 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_left_local_05066); + +CREATE TABLE t_right_local_05066 (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_right_local_05066 VALUES (1, 10), (2, 20), (3, 30); + +CREATE TABLE t_right_dist_05066 +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x * 2 +) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_right_local_05066); + +SELECT 'projected_marker_on'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +-- The projection path also sets an alias on the marker. Referencing the column only in a clause +-- exercises the path where no alias is set at all. +SELECT 'clause_only_marker_on'; +SELECT m.id +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +WHERE r.foo > 30 +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'projected_marker_off'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +SELECT 'clause_only_marker_off'; +SELECT m.id +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +WHERE r.foo > 30 +ORDER BY m.id +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +-- The same rewrite reaches a plain `JOIN` through `prefer_global_in_and_join`. +SELECT 'implicit_global_join'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, prefer_global_in_and_join = 1; + +DROP TABLE t_right_dist_05066; +DROP TABLE t_right_local_05066; +DROP TABLE t_left_dist_05066; +DROP TABLE t_left_local_05066; From 63e9fed077177b3a10365662137a671bdb1e6fde Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 19 Sep 2026 09:53:22 +0200 Subject: [PATCH 08/18] Revert two StorageMerge changes that do nothing Both were argued from cases that cannot occur, and both sit in a path every `Merge` table with child `ALIAS` columns goes through. `use_column_identifier_as_action_node_name` goes back to `false`. It was set to `true` so that an `ALIAS` defined over another one would resolve against the child stream rather than be expanded again, because `false` makes `PlannerActionsVisitor` replace a column node that has an expression with a visit of that expression. No such column node reaches it. `ApplyAliasColumnExpressionsVisitor` runs over each alias before it is recorded, replacing every column node that has an expression with the expression itself, and the visitor descends into what it substituted -- so `b ALIAS a + 1` over `a ALIAS x + 1` is already flattened to physical columns by the time `convertAndFilterSourceStream` re-analyzes it. The naming half of the setting is inert too: no column identifiers are registered for that table expression, so the action node name falls back to the plain column name whichever value is passed. The plain-name to identifier mapping goes back to taking the first matching declared column instead of the longest. The longest match was meant to stop a table declaring both `b` and `` `a.b` `` from letting `b` claim `__table1.a.b`. Nothing produces that name: `buildColumnIdentifier` always backquotes a dotted column name, so the identifier is ``__table1.`a.b` `` and does not end in `.b`. Reintroducing the first-match behaviour leaves `05065` passing, which is what settled it. The clause accepting an unquoted dotted name stays, now described as defensive rather than as something a branch emits. `05065` stays as a values guard for a schema shape nothing else covers, with its header cut down to say that and nothing more. Everything else from the `Merge` commit stands: the plain-name aliasing loop, the identifier-named alias outputs and the declared-schema mapping are the actual fix, and `04281`, `05057` and `05059` cover them. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- src/Storages/StorageMerge.cpp | 34 ++++++------------- ...ed_dotted_alias_merge_over_distributed.sql | 19 ++--------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index c606d8cc3e6a..5fb2af1552dc 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1708,29 +1708,23 @@ void ReadFromMerge::convertAndFilterSourceStream( } /// Otherwise look for `.` or ``.`` `` where C is a declared Merge column. Columns - /// matching no declared name are skipped; they are intermediate outputs of the child plan. Branches - /// differ in how they quote a dotted name -- some emit ``__tableN.`n.a` ``, others `__tableN.n.a` -- so - /// accept both. + /// matching no declared name are skipped; they are intermediate outputs of the child plan. /// - /// Take the longest matching name rather than the first. A table declaring both `b` and `` `a.b` `` - /// makes `__table1.a.b` end with `.b` as well, so a first match in schema order could hand `b` the - /// wrong column. Nothing reaches that state today -- an analyzer identifier backquotes a dotted name, - /// so the raw form only arrives from a child that never emits a dotted identifier at all -- and the - /// longest match is what keeps the mapping from depending on that. - const String * best_match = nullptr; + /// The unquoted form of a dotted name is accepted defensively. No producer of it is known: + /// `buildColumnIdentifier` always backquotes a dotted column name, so an analyzer identifier for `n.a` + /// is always ``__tableN.`n.a` ``. for (const auto & merge_column : all_merge_columns) { const bool dotted = merge_column.name.find('.') != String::npos; const String want_raw = "." + merge_column.name; const String want_quoted = dotted ? ("." + backQuote(merge_column.name)) : want_raw; - if (!column.name.ends_with(want_quoted) && !(dotted && column.name.ends_with(want_raw))) - continue; - if (!best_match || best_match->size() < merge_column.name.size()) - best_match = &merge_column.name; + if (column.name.ends_with(want_quoted) || (dotted && column.name.ends_with(want_raw))) + { + if (!plain_to_identifier.emplace(merge_column.name, column.name).second) + ambiguous.insert(merge_column.name); + break; + } } - - if (best_match && !plain_to_identifier.emplace(*best_match, column.name).second) - ambiguous.insert(*best_match); } for (const auto & ambiguous_name : ambiguous) plain_to_identifier.erase(ambiguous_name); @@ -1777,13 +1771,7 @@ void ReadFromMerge::convertAndFilterSourceStream( collectSets(query_tree, *modified_query_info.planner_context); ColumnNodePtrWithHashSet empty_correlated_columns_set; - /// `true` not for the naming it implies -- no column identifiers are registered for this table expression, - /// so `calculateActionNodeName` falls back to the plain column name either way -- but for what `false` - /// does to an `ALIAS` column: it replaces the column node with a visit of its defining expression. An - /// `ALIAS` defined over another one (`b ALIAS a + 1` over `a ALIAS x + 1`) would then expand `a` again - /// rather than read the value the child stream already carries, and the physical columns underneath it - /// need not be in that stream at all. The plain-name aliases added above are what the lookup then hits. - PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, true /*use_column_identifier_as_action_node_name*/); + PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, empty_correlated_columns_set, false /*use_column_identifier_as_action_node_name*/); const auto & [nodes, _] = actions_visitor.visit(actions_dag, query_tree); if (nodes.size() != 1) diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql index 34310f184503..79a985ba163d 100644 --- a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql @@ -1,19 +1,6 @@ --- End-to-end guard for a `Merge` over `Distributed` whose schema declares both a plain column and a --- dotted column whose tail is that plain name (`b` and `` `a.b` ``), read as `ALIAS` columns. --- --- What this does NOT do: reproduce the first-match bug in the name mapping of --- `ReadFromMerge::convertAndFilterSourceStream`. That mapping matches declared Merge column names --- against the suffixes of the child stream's column names, and `__table1.a.b` ends with `.b`, so a --- first match in schema order would hand `b` the wrong column. No input can put it in that state: --- the Merge target header carries analyzer identifiers, which backquote a dotted name --- (`__table1.`a.b``, `PlannerContext.cpp` `buildColumnIdentifier`), and the child stream carries the --- alias values under the alias name or the expression name, never a dotted analyzer identifier, --- because the query reaching the child has every Merge-level `ALIAS` reference already replaced by --- its expression. Verified by reintroducing the first-match behaviour and observing this test still --- pass. --- --- So it asserts the values are right for a schema shape nothing else covers. 05059 covers dotted --- names on their own. +-- Values guard for a `Merge` over `Distributed` whose schema declares both a plain column and a +-- dotted column whose tail is that plain name (`b` and `` `a.b` ``), both read as `ALIAS` columns. +-- 05059 covers dotted names on their own; this shape is covered nowhere else. -- -- `test_cluster_two_shards` reads the local table twice, so every query groups to dedup. From 9cdf271fb7dcb313321e8ce2fe5b2db247b407f2 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 19 Sep 2026 12:00:49 +0200 Subject: [PATCH 09/18] docs: describe __aliasMarker as it actually behaves `__aliasMarker` stopped requiring a `String` second argument when the id became a live `ColumnNode`, but its registration still declared one. Altinity#1844 made the same relaxation and edited only the one-line description, and this branch inherited that, so the documented contract has been wrong since the marker gained its second phase. The argument is an identity rather than a name, so call it `alias_id`, and say what it is for: recording which `ALIAS` column an expression was inlined from, so that a shard names the result the way the initiator expects. Direct use is still not intended, but it is harmless -- the function returns its first argument unchanged. `03648_alias_marker_with_mergeable_state.sh` gains a note on why every invocation passes `--query_kind secondary_query`. The test prints the header a shard leg produces, and a query marked as secondary skips AST-level optimizations and splits aggregation differently, neither of which a real shard leg would have applied. It came from Altinity#1844 unchanged and nothing recorded the reason. Related: https://github.com/Altinity/ClickHouse/pull/1844 Co-Authored-By: Claude Opus 5 --- src/Functions/identity.cpp | 8 +++++--- .../03648_alias_marker_with_mergeable_state.sh | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index 05d2ef870601..5245b39921b0 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -38,12 +38,14 @@ REGISTER_FUNCTION(AliasMarker) { factory.registerFunction(FunctionDocumentation{ .description = R"( -Internal function that marks ALIAS column expressions for the analyzer. Not intended for direct use. +Internal function. Returns its first argument unchanged. The second argument records which ALIAS column the +expression was inlined from, so that a shard names the result the way the initiator expects. Not intended for +direct use, but harmless when used directly. )", - .syntax = {"__aliasMarker(expr, alias_name)"}, + .syntax = {"__aliasMarker(expr, alias_id)"}, .arguments = { {"expr", "Expression to mark.", {"Any"}}, - {"alias_name", "Alias name attached to the expression.", {"String"}}, + {"alias_id", "Identity of the ALIAS column the expression came from.", {"Any"}}, }, .returned_value = {"Returns expr unchanged.", {"Any"}}, .introduced_in = {25, 8}, diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh index fb0580e796f0..1125a06b3d54 100755 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh @@ -4,6 +4,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +# `--query_kind secondary_query` goes with every `--stage` below, because the point is to print the +# header a shard leg produces. A query marked as secondary skips AST-level optimizations +# (`PlannerContext.cpp`, `is_ast_level_optimization_allowed`) and changes how aggregation is split, +# neither of which a real shard leg would have applied. Inherited unchanged from Altinity#1844. + echo "---- stage: with_mergeable_state (analyzer=1, setting=enable_alias_marker=1) ----" $CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state --multiquery 2>&1 <<'EOF' | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' SET enable_alias_marker=1; From eb13e8b3fb10f577682b8ae586c126a339ace31c Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 26 Sep 2026 08:52:27 +0200 Subject: [PATCH 10/18] tests: skip `05060` in the fast test The fast test fails `05060_hybrid_unknown_table_exact_schema` with `Unknown table engine IcebergS3` (`UNKNOWN_STORAGE`). The fast test build has no S3 or Iceberg support, and the test creates an `IcebergS3` table. Tag it `no-fasttest`, like the other Iceberg tests. The note about `enable_parallel_blocks_marshalling = 0` goes too: the `DISTINCT` over `ColumnBLOB` abort it worked around is fixed by https://github.com/ClickHouse/ClickHouse/pull/111997, now merged here. Co-Authored-By: Claude Opus 5.5 --- .../0_stateless/05060_hybrid_unknown_table_exact_schema.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql index 55d247058b3f..d899f084ba4c 100644 --- a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql @@ -1,3 +1,4 @@ +-- Tags: no-fasttest -- Hybrid table joined against a MergeTree and an Iceberg table, covering Altinity#1208, #1209 and -- #1422. Both `object_storage_cluster_join_mode` values must return the same rows: `'local'` used -- to raise UNKNOWN_IDENTIFIER here, and no longer does, so it is held to the `'allow'` result. @@ -6,8 +7,6 @@ -- 127.0.0.1 and 127.0.0.2, because `icebergCluster` below reads through -- `test_cluster_one_shard_three_replicas_localhost`. Without the third address the query still -- returns the right rows by failing over, but the connection warnings on stderr fail the test. --- Run with `enable_parallel_blocks_marshalling = 0` until the DISTINCT-over-ColumnBLOB abort is --- fixed. SET allow_experimental_hybrid_table = 1, enable_analyzer = 1, prefer_localhost_replica = 0, From cceecd5d9791e40f23fde944728eb6c8c041f923 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 26 Sep 2026 13:47:30 +0200 Subject: [PATCH 11/18] Fix provenance of injected `__aliasMarker` in distributed queries Use a three-argument pending form only for injected markers and finalize it to the existing two-argument string-ID form before shipping. Leave hand-written two-argument calls subject to normal alias inlining and column collection, and reject pending markers at SQL and plan shipping entry points. Keep the pending token literal during long-constant rewriting. Cover direct calls, temporary-table dependencies, finalized shard SQL, and the intermediate `JOIN USING` projection-alias shape with functional and unit tests. Older shards may still reject hand-written two-argument calls whose ID is a column rather than a string. Injected markers retain the existing shard SQL form. Tests: `05067`-`05071`, `AliasMarkerPending.*`, and the existing alias-marker regression suite. --- src/Analyzer/Utils.cpp | 48 +++++-- src/Analyzer/Utils.h | 11 +- src/Functions/identity.cpp | 2 +- src/Functions/identity.h | 42 +++--- .../ClusterProxy/executeQuery.cpp | 3 + src/Planner/PlannerActionsVisitor.cpp | 29 ++-- src/Planner/Utils.cpp | 3 + .../QueryPlan/ParallelReplicasLocalPlan.cpp | 4 + src/Storages/IStorageCluster.cpp | 5 + src/Storages/buildQueryTreeForShard.cpp | 43 ++++-- .../tests/gtest_alias_marker_pending.cpp | 131 ++++++++++++++++++ ...rker_direct_distributed_identity.reference | 28 ++++ ...ias_marker_direct_distributed_identity.sql | 36 +++++ ...as_marker_direct_id_dependencies.reference | 40 ++++++ ...68_alias_marker_direct_id_dependencies.sql | 63 +++++++++ ...ias_marker_projection_join_using.reference | 12 ++ ...069_alias_marker_projection_join_using.sql | 32 +++++ ..._alias_marker_pending_boundaries.reference | 8 ++ .../05070_alias_marker_pending_boundaries.sql | 36 +++++ ...71_alias_marker_shipped_sql_form.reference | 4 + .../05071_alias_marker_shipped_sql_form.sql | 48 +++++++ 21 files changed, 564 insertions(+), 64 deletions(-) create mode 100644 src/Storages/tests/gtest_alias_marker_pending.cpp create mode 100644 tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.reference create mode 100644 tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.sql create mode 100644 tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.reference create mode 100644 tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.sql create mode 100644 tests/queries/0_stateless/05069_alias_marker_projection_join_using.reference create mode 100644 tests/queries/0_stateless/05069_alias_marker_projection_join_using.sql create mode 100644 tests/queries/0_stateless/05070_alias_marker_pending_boundaries.reference create mode 100644 tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql create mode 100644 tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.reference create mode 100644 tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.sql diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index cc9f77498055..eb44eff6376c 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -31,6 +31,7 @@ #include #include +#include #include @@ -55,6 +56,7 @@ #include #include +#include namespace DB { @@ -1035,30 +1037,27 @@ class FinalizeAliasMarkersVisitor : public InDepthQueryTreeVisitorgetArguments().getNodes(); - if (arguments.size() != 2 || !arguments[0] || !arguments[1]) + if (arguments.size() != 3) return; - /// Already materialized on an earlier hop. - if (const auto * id_node = arguments[1]->as(); id_node && isString(id_node->getResultType())) - return; + if (!arguments[0] || !arguments[1] || !arguments[2]) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid internal pending __aliasMarker arguments"); + + const auto * token = arguments[2]->as(); + if (!token || !isString(token->getResultType()) || token->getValue().safeGet() != AliasMarkerName::pending_token) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid internal pending __aliasMarker token"); const auto * column_node = arguments[1]->as(); if (!column_node) - return; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Internal pending __aliasMarker requires a column id"); const auto & column_source = column_node->getColumnSourceOrNull(); - if (!column_source) - return; - - /// A lambda parameter -- `arrayMap(x -> __aliasMarker(x, x), ...)` written by hand -- has the `LambdaNode` as - /// its source. There is no table alias to build an id from, and the marker is a per-row identity rather than a - /// transport marker, so leave it as it is. A marker this pass injected inside a lambda body does not land here: - /// its id names an `ALIAS` column of a table, and the table expression is its source. - if (column_source->getNodeType() == QueryTreeNodeType::LAMBDA || !column_source->hasAlias()) - return; + if (!column_source || column_source->getNodeType() == QueryTreeNodeType::LAMBDA || !column_source->hasAlias()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Internal pending __aliasMarker id has no table source alias"); auto alias_id = column_source->getAlias() + "." + column_node->getColumnName(); arguments[1] = std::make_shared(std::move(alias_id), std::make_shared()); + arguments.pop_back(); resolveOrdinaryFunctionNodeByName(*function_node, "__aliasMarker", context); } @@ -1074,6 +1073,27 @@ void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, co visitor.visit(node); } +void assertNoPendingAliasMarkersForDistributedSerialization(const QueryTreeNodePtr & node) +{ + if (!node) + return; + + std::vector nodes_to_visit{node.get()}; + while (!nodes_to_visit.empty()) + { + const auto * current = nodes_to_visit.back(); + nodes_to_visit.pop_back(); + + if (const auto * function_node = current->as(); + function_node && function_node->getFunctionName() == "__aliasMarker" && function_node->getArguments().getNodes().size() == 3) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Internal pending __aliasMarker cannot be shipped before finalization"); + + for (const auto & child : current->getChildren()) + if (child) + nodes_to_visit.push_back(child.get()); + } +} + std::pair getExpressionSource(const QueryTreeNodePtr & node) { if (const auto * column = node->as()) diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 08cf5215426a..843d0992d646 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -157,19 +157,22 @@ void resolveOrdinaryFunctionNodeByName(FunctionNode & function_node, const Strin /// Arguments and parameters are taken from the node. void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const String & function_name); -/** Materialize the id argument of every `__aliasMarker` in the tree, turning the `ColumnNode` it carries into the - * `String` constant the shard will read. +/** Materialize the id of each injected, three-argument pending `__aliasMarker` in the tree. Replace its `ColumnNode` + * with the `String` constant the shard will read, and remove its pending-form token. Leave two-argument calls alone. * * Call this immediately before the tree is rendered as SQL for a shard, and no earlier. The id is the column's * analyzer identifier, and `createUniqueAliasesIfNecessary` -- which runs inside `buildQueryTreeForShard` -- is what * settles the `__tableN` part of it. An id frozen before that point names a table alias that no longer exists by the * time the shard header is built. * - * A marker whose id is already a `String` constant was materialized on an earlier hop and is left alone. A marker - * carrying anything else is a user-written one, not ours, and is also left alone. + * A marker already materialized on an earlier hop has two arguments. A hand-written two-argument marker is left + * alone too, even when its id is a `ColumnNode`. */ void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, const ContextPtr & context); +/// Reject a pending `__aliasMarker` before converting a query tree to SQL or building a plan to ship. +void assertNoPendingAliasMarkersForDistributedSerialization(const QueryTreeNodePtr & node); + /// Returns single source of expression node. /// First element of pair is source node, can be nullptr if there are no sources or multiple sources. /// Second element of pair is true if there is at most one source, false if there are multiple sources. diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index 5245b39921b0..8aeed5bbacd8 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -40,7 +40,7 @@ REGISTER_FUNCTION(AliasMarker) .description = R"( Internal function. Returns its first argument unchanged. The second argument records which ALIAS column the expression was inlined from, so that a shard names the result the way the initiator expects. Not intended for -direct use, but harmless when used directly. +direct use: an explicitly supplied string id controls the planner's action name. )", .syntax = {"__aliasMarker(expr, alias_id)"}, .arguments = { diff --git a/src/Functions/identity.h b/src/Functions/identity.h index 8278fff52af7..a5483624377a 100644 --- a/src/Functions/identity.h +++ b/src/Functions/identity.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -106,6 +107,7 @@ class FunctionActionName final : public FunctionIdentityBase struct AliasMarkerName { static constexpr auto name = "__aliasMarker"; + static constexpr auto pending_token = "__aliasMarker_pending_v1"; }; /** `__aliasMarker(expr, id)` is an internal pass-through identity. It returns `expr` untouched; `id` exists only to @@ -124,15 +126,13 @@ struct AliasMarkerName * with a forced result name; `__aliasMarker` is consumed into an alias on top of its child, which is what keeps the * expression behaving like a distinct logical column. * - * The second argument travels in two forms. Between injection and serialization it is a `ColumnNode`, so the ordinary - * analyzer passes keep transforming it along with everything else -- in particular `createUniqueAliasesIfNecessary`, - * which assigns the final `__tableN` aliases. `finalizeAliasMarkersForDistributedSerialization` then materializes it - * into a `String` constant just before the query is rendered as SQL. Freezing the id any earlier captures a table - * alias that later changes, and the shard then emits a name the initiator never asked for. + * An injected marker has a third, constant token argument until it is shipped. The second argument is a `ColumnNode` + * so analyzer passes, in particular `createUniqueAliasesIfNecessary`, can assign the final `__tableN` alias. + * `finalizeAliasMarkersForDistributedSerialization` replaces it with a `String` id and removes the token before shipping. + * The token distinguishes this pending form from a hand-written two-argument call; it is not authentication. * - * Both forms are accepted here, and so is anything else: a user can write `__aliasMarker(x, x)` and it behaves as an - * identity. Rejecting unexpected arguments in the function would turn user SQL into a server-side error for no gain, - * since the marker is harmless by construction. + * A hand-written two-argument `__aliasMarker(x, x)` is not finalized. An explicitly supplied `String` id still + * controls action naming, so direct use of this internal function is not guaranteed to be harmless. * * This is a bridge for as long as distributed transport still goes through SQL text. Once query plan serialization * replaces that boundary, the marker should become unnecessary. @@ -144,19 +144,29 @@ class FunctionAliasMarker : public IFunction static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 2; } - /// The id argument is a constant only after finalization. Before that it is a `ColumnNode`, and between the two - /// the function must still resolve. - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + /// Index 2 exists only on the pending form; the executable and DAG builder ignore absent constant arguments. + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; } bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + /// Validate the token even if an argument is `NULL` or has type `Nothing`. + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForNothing() const override { return false; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); + if (arguments.size() != 2 && arguments.size() != 3) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments, or 3 for its internal pending form"); - return arguments.front(); + if (arguments.size() == 3) + { + const auto * token = arguments[2].column ? checkAndGetColumn(arguments[2].column.get()) : nullptr; + if (!WhichDataType(arguments[2].type).isString() || !token || token->getValue() != AliasMarkerName::pending_token) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid internal pending __aliasMarker token"); + } + + return arguments.front().type; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 89e80bb9cb7a..03fa1f1a1b75 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -1169,6 +1170,8 @@ std::optional executeInsertSelectWithParallelReplicas( { InterpreterSelectQueryAnalyzer analyzer(query_ast.select, new_context, {}); const auto & query_tree = analyzer.getQueryTree(); + /// This parallel `INSERT SELECT` path renders directly, bypassing `queryNodeToDistributedSelectQuery`. + assertNoPendingAliasMarkersForDistributedSerialization(query_tree); auto select_ast = query_tree->toAST(); auto new_query_ast = query_ast.clone(); diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 3b2fb3c78fa9..48121345da44 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -95,8 +95,8 @@ String calculateActionNodeNameWithCastIfNeeded(const ConstantNode & constant_nod return buffer.str(); } -/// Return a `__aliasMarker`'s finalized id, or an empty string when it has none yet. Between injection and -/// serialization the id is a `ColumnNode`, and a user-written marker can hold anything at all. +/// Return a two-argument `__aliasMarker`'s string id, or an empty string when it has none. +/// A three-argument pending marker always resolves to its payload in an intermediate plan. String tryExtractAliasMarkerId(const QueryTreeNodePtr & id_argument) { if (const auto * id_node = id_argument->as(); id_node && isString(id_node->getResultType())) @@ -209,16 +209,14 @@ class ActionNodeNameHelper if (function_node.getFunctionName() == "__aliasMarker") { const auto & function_argument_nodes = function_node.getArguments().getNodes(); - if (function_argument_nodes.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); + if (function_argument_nodes.size() != 2 && function_argument_nodes.size() != 3) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 or 3 arguments"); - result = tryExtractAliasMarkerId(function_argument_nodes.at(1)); + if (function_argument_nodes.size() == 2) + result = tryExtractAliasMarkerId(function_argument_nodes.at(1)); - /// No finalized id: either the marker has not reached its serialization boundary yet, or a user - /// wrote one by hand. Name the node after the expression it wraps, which is what the name would - /// have been without a marker at all. The two cases are not distinguishable here -- a hand-written - /// `__aliasMarker(x, x)` looks exactly like an injected one whose id is still a `ColumnNode` -- so - /// this cannot be turned into an assertion without rejecting valid SQL (03933). + /// A pending marker, or a hand-written one without a string id, is named after its payload. + /// The pending form can reach an intermediate planner, but must be finalized before shipping. if (result.empty()) result = calculateActionNodeName(function_argument_nodes.at(0)); @@ -1269,14 +1267,15 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi if (function_node.getFunctionName() == "__aliasMarker") { const auto & function_arguments = function_node.getArguments().getNodes(); - if (function_arguments.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); + if (function_arguments.size() != 2 && function_arguments.size() != 3) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 or 3 arguments"); auto [child_name, levels] = visitImpl(function_arguments.at(0)); - /// Without a finalized id the marker adds no identity of its own, so it resolves to its payload. That happens - /// for a marker the user wrote by hand, and for one that has not reached a serialization boundary yet. - auto alias_id = tryExtractAliasMarkerId(function_arguments.at(1)); + /// An intermediate pending marker and a hand-written marker without a string id resolve to their payload. + String alias_id; + if (function_arguments.size() == 2) + alias_id = tryExtractAliasMarkerId(function_arguments.at(1)); if (alias_id.empty()) alias_id = child_name; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index f48515204da2..dcfe1e5476fe 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -317,6 +317,9 @@ void normalizeAliasMarkersInQueryTree(QueryTreeNodePtr & node) ASTPtr queryNodeToDistributedSelectQuery(const QueryTreeNodePtr & query_node) { + /// Check before normalization: it can discard a nested marker and hide an unfinalized pending id. + assertNoPendingAliasMarkersForDistributedSerialization(query_node); + /// Remove CTEs information from distributed queries. /// Now, if cte_name is set for subquery node, AST -> String serialization will only print cte name. /// But CTE is defined only for top-level query part, so may not be sent. diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index d63fd813be14..a88baa30a992 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -116,6 +117,9 @@ std::shared_ptr createRemotePlanForParallelReplicas( { checkStackSize(); + /// A pending marker planned here would resolve to its payload and disappear before the plan was shipped. + assertNoPendingAliasMarkersForDistributedSerialization(query_tree); + auto new_context = Context::createCopy(context); auto select_query_options = SelectQueryOptions(processed_stage); diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index fe1a51a861c0..83c756ff39c3 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -437,6 +437,11 @@ void IStorageCluster::read( auto cluster_name_from_settings = getClusterName(context); const auto & settings = context->getSettingsRef(); + /// The ALLOW path can forward the original AST without calling `queryNodeToDistributedSelectQuery`. + if (settings[Setting::allow_experimental_analyzer] + && (!cluster_name_from_settings.empty() || settings[Setting::object_storage_remote_initiator])) + assertNoPendingAliasMarkersForDistributedSerialization(query_info.query_tree); + ASTPtr query_to_send = query_info.query; if (cluster_name_from_settings.empty()) diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 0dfdb324db2a..1b4c5a82e052 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -109,20 +110,27 @@ QueryTreeNodePtr getInlineableAliasColumnExpression(const QueryTreeNodePtr & nod QueryTreeNodePtr wrapInAliasMarker(QueryTreeNodePtr expression, const ColumnNode & column_node, const ContextPtr & context) { auto marker_id = std::make_shared(column_node.getColumn(), column_node.getColumnSourceOrNull()); + auto token = std::make_shared(String(AliasMarkerName::pending_token), std::make_shared()); if (auto * function_node = expression->as(); function_node && function_node->getFunctionName() == "__aliasMarker") { auto & marker_arguments = function_node->getArguments().getNodes(); - if (marker_arguments.size() == 2) + if (marker_arguments.size() == 2 || marker_arguments.size() == 3) + { marker_arguments[1] = std::move(marker_id); - return expression; + marker_arguments.resize(2); + marker_arguments.emplace_back(std::move(token)); + resolveOrdinaryFunctionNodeByName(*function_node, "__aliasMarker", context); + return expression; + } } QueryTreeNodes arguments; - arguments.reserve(2); + arguments.reserve(3); arguments.emplace_back(std::move(expression)); arguments.emplace_back(std::move(marker_id)); + arguments.emplace_back(std::move(token)); auto marker_node = std::make_shared("__aliasMarker"); marker_node->getArguments().getNodes() = std::move(arguments); @@ -179,11 +187,10 @@ struct AliasColumnInliner marker_node && marker_node->getFunctionName() == "__aliasMarker") { auto & marker_arguments = marker_node->getArguments().getNodes(); - if (marker_arguments.size() == 2) + if (marker_arguments.size() == 3) { - /// Descend into the payload only. The second argument is the column reference the marker exists to - /// record, and inlining it would expand the very `ALIAS` column whose identity it carries. It has to - /// reach `finalizeAliasMarkersForDistributedSerialization` as a `ColumnNode`. + /// The pending form's second argument is an injected column identity, not a column read by the query. + /// A hand-written two-argument marker is traversed normally: its id must resolve in the shipped SQL. inlineExpression(marker_arguments[0]); return; } @@ -317,18 +324,16 @@ class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitoras(); function_node && function_node->getFunctionName() == "__aliasMarker") { const auto & marker_arguments = function_node->getArguments().getNodes(); - if (marker_arguments.size() == 2 && marker_arguments[1]) + if (marker_arguments.size() == 3 && marker_arguments[1]) marker_id_nodes.insert(marker_arguments[1].get()); return; } @@ -575,6 +580,14 @@ class ReplaceLongConstWithScalarVisitor : public InDepthQueryTreeVisitorWithCont void enterImpl(QueryTreeNodePtr & node) { + if (const auto * function_node = node->as(); + function_node && function_node->getFunctionName() == "__aliasMarker") + { + const auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() == 3 && arguments[2]) + pending_marker_tokens.insert(arguments[2].get()); + } + // Do not visit second argument of "in" functions if (!in_second_argument.empty() && in_second_argument.top() == node) { @@ -590,7 +603,8 @@ class ReplaceLongConstWithScalarVisitor : public InDepthQueryTreeVisitorWithCont auto * constant_node = node->as(); - if (!constant_node) + /// Keep pending tokens literal until the finalizer validates and removes them. + if (!constant_node || pending_marker_tokens.contains(node.get())) return; const auto * col_const = typeid_cast(constant_node->getColumn().get()); @@ -632,6 +646,7 @@ class ReplaceLongConstWithScalarVisitor : public InDepthQueryTreeVisitorWithCont private: Int64 max_size = 0; std::stack in_second_argument; + std::unordered_set pending_marker_tokens; }; // Helper function to add DISTINCT to all QueryNode objects inside a query/union subtree diff --git a/src/Storages/tests/gtest_alias_marker_pending.cpp b/src/Storages/tests/gtest_alias_marker_pending.cpp new file mode 100644 index 000000000000..f96fc2ea5c0c --- /dev/null +++ b/src/Storages/tests/gtest_alias_marker_pending.cpp @@ -0,0 +1,131 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} +} + +using namespace DB; + +namespace +{ + +class MarkerTestTableNode final : public IQueryTreeNode +{ +public: + MarkerTestTableNode() : IQueryTreeNode(0) {} + + QueryTreeNodeType getNodeType() const override { return QueryTreeNodeType::TABLE; } + void dumpTreeImpl(WriteBuffer &, FormatState &, size_t) const override {} + bool isEqualImpl(const IQueryTreeNode &, CompareOptions) const override { return true; } + void updateTreeHashImpl(HashState &, CompareOptions) const override {} + QueryTreeNodePtr cloneImpl() const override { return std::make_shared(); } + ASTPtr toASTImpl(const ConvertToASTOptions &) const override { return nullptr; } +}; + +FunctionNodePtr makeMarker(QueryTreeNodes arguments) +{ + auto marker = std::make_shared(AliasMarkerName::name); + marker->getArguments().getNodes() = std::move(arguments); + return marker; +} + +} + +TEST(AliasMarkerPending, RejectNestedMarkerBeforeShipping) +{ + auto pending = makeMarker({std::make_shared(1), std::make_shared(1), + std::make_shared(String(AliasMarkerName::pending_token), std::make_shared())}); + auto outer = makeMarker({pending, std::make_shared(String("outer"), std::make_shared())}); + auto query = std::make_shared(Context::createCopy(getContext().context)); + query->getProjection().getNodes().push_back(outer); + QueryTreeNodePtr root = query; + + auto expect_pending_rejected = [](auto && ship) + { + try + { + ship(); + FAIL() << "A nested pending marker must not cross the shipping boundary"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + EXPECT_NE(e.message().find("cannot be shipped before finalization"), String::npos); + } + }; + + /// Both shipping entry points check before normalization or planning can discard the inner marker. + expect_pending_rejected([&] { assertNoPendingAliasMarkersForDistributedSerialization(root); }); + expect_pending_rejected([&] { (void)queryNodeToDistributedSelectQuery(root); }); + expect_pending_rejected([&] + { + (void)createRemotePlanForParallelReplicas(root, Block{}, query->getContext(), QueryProcessingStage::FetchColumns); + }); +} + +TEST(AliasMarkerPending, ProjectionAliasOnJoinUsingSide) +{ + tryRegisterFunctions(); + auto context = Context::createCopy(getContext().context); + context->setSetting("enable_alias_marker", Field(1)); + context->setSetting("analyzer_compatibility_join_using_top_level_identifier", Field(1)); + + auto left = std::make_shared(); + left->setAlias("__table1"); + auto right = std::make_shared(); + right->setAlias("__table2"); + + const auto type = std::make_shared(); + auto column = std::make_shared(NameAndTypePair("x", type), left); + /// Model the expression-bearing column that projection-alias resolution places on the left `USING` side. + auto projection_alias = std::make_shared(NameAndTypePair("k", type), column, left); + auto right_key = std::make_shared(NameAndTypePair("k", type), right); + auto key_sides = std::make_shared(QueryTreeNodes{projection_alias, right_key}); + auto using_key = std::make_shared(NameAndTypePair("k", type), key_sides, left); + auto using_list = std::make_shared(QueryTreeNodes{using_key}); + + auto join = std::make_shared(left, right, using_list, + JoinLocality::Unspecified, JoinStrictness::All, JoinKind::Inner, true); + auto query = std::make_shared(context); + query->getJoinTree() = join; + QueryTreeNodePtr root = query; + + inlineAliasColumns(root, context); + + const auto & side = key_sides->getNodes()[0]; + const auto * pending = side->as(); + ASSERT_NE(pending, nullptr); + ASSERT_EQ(pending->getArguments().getNodes().size(), 3); + EXPECT_EQ(pending->getArguments().getNodes()[1]->as().getColumnName(), "k"); + EXPECT_THROW(assertNoPendingAliasMarkersForDistributedSerialization(root), Exception); + + finalizeAliasMarkersForDistributedSerialization(root, context); + + const auto & finalized_arguments = pending->getArguments().getNodes(); + ASSERT_EQ(finalized_arguments.size(), 2); + EXPECT_EQ(finalized_arguments[1]->as().getValue().safeGet(), "__table1.k"); + EXPECT_NO_THROW(assertNoPendingAliasMarkersForDistributedSerialization(root)); +} diff --git a/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.reference b/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.reference new file mode 100644 index 000000000000..2ad1f43be009 --- /dev/null +++ b/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.reference @@ -0,0 +1,28 @@ +local +4 +2 +3 +4 +5 +6 +distributed_marker_on +4 +2 +3 +4 +5 +6 +distributed_marker_off +4 +2 +3 +4 +5 +6 +distributed_serialized_plan +4 +2 +3 +4 +5 +6 diff --git a/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.sql b/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.sql new file mode 100644 index 000000000000..8d11e82482f3 --- /dev/null +++ b/tests/queries/0_stateless/05067_alias_marker_direct_distributed_identity.sql @@ -0,0 +1,36 @@ +-- A direct two-argument `__aliasMarker` must not acquire a transport ID from its column argument. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_local_05067; +DROP TABLE IF EXISTS t_dist_05067; + +CREATE TABLE t_local_05067 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_05067 VALUES (1), (2), (3), (4), (5); + +CREATE TABLE t_dist_05067 AS t_local_05067 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05067); + +SELECT 'local'; +SELECT count() FROM t_local_05067 WHERE __aliasMarker(x + 1, x) > 2; +SELECT __aliasMarker(x + 1, x) FROM t_local_05067 ORDER BY x; + +SELECT 'distributed_marker_on'; +SELECT count() FROM t_dist_05067 WHERE __aliasMarker(x + 1, x) > 2 +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, x) FROM t_dist_05067 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'distributed_marker_off'; +SELECT count() FROM t_dist_05067 WHERE __aliasMarker(x + 1, x) > 2 +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, x) FROM t_dist_05067 ORDER BY x +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +SELECT 'distributed_serialized_plan'; +SELECT count() FROM t_dist_05067 WHERE __aliasMarker(x + 1, x) > 2 +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; +SELECT __aliasMarker(x + 1, x) FROM t_dist_05067 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_05067; +DROP TABLE t_local_05067; diff --git a/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.reference b/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.reference new file mode 100644 index 000000000000..5e2db433c5d3 --- /dev/null +++ b/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.reference @@ -0,0 +1,40 @@ +alias_on_both_marker_on +4 +2 +3 +4 +5 +6 +alias_on_both_marker_off +4 +2 +3 +4 +5 +6 +alias_only_on_distributed_marker_on +4 +2 +3 +4 +5 +6 +alias_only_on_distributed_marker_off +4 +2 +3 +4 +5 +6 +pending_token_survives_scalar_rewrite +10 +20 +30 +40 +50 +global_join_id_only +1 1 +2 1 +3 1 +4 1 +5 1 diff --git a/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.sql b/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.sql new file mode 100644 index 000000000000..a36559ce02ca --- /dev/null +++ b/tests/queries/0_stateless/05068_alias_marker_direct_id_dependencies.sql @@ -0,0 +1,63 @@ +-- A hand-written marker's second argument is still a SQL column reference until the shard analyzes it. +-- It must be inlined when it is an `ALIAS` and collected when it is needed by a `GLOBAL JOIN` temporary table. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_local_05068; +DROP TABLE IF EXISTS t_dist_05068; +DROP TABLE IF EXISTS t_dist_only_alias_05068; +DROP TABLE IF EXISTS t_left_05068; +DROP TABLE IF EXISTS t_right_05068; + +CREATE TABLE t_local_05068 (x UInt64, computed UInt64 ALIAS x * 10) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_05068 (x) VALUES (1), (2), (3), (4), (5); +CREATE TABLE t_dist_05068 AS t_local_05068 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05068); + +CREATE TABLE t_left_05068 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_left_05068 VALUES (1), (2), (3), (4), (5); +-- This alias exists only on the initiator's `Distributed` schema, not on the local storage. +CREATE TABLE t_dist_only_alias_05068 (x UInt64, computed UInt64 ALIAS x * 10) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_left_05068); + +SELECT 'alias_on_both_marker_on'; +SELECT count() FROM t_dist_05068 WHERE __aliasMarker(x + 1, computed) > 2 +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, computed) FROM t_dist_05068 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; +SELECT 'alias_on_both_marker_off'; +SELECT count() FROM t_dist_05068 WHERE __aliasMarker(x + 1, computed) > 2 +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, computed) FROM t_dist_05068 ORDER BY x +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +SELECT 'alias_only_on_distributed_marker_on'; +SELECT count() FROM t_dist_only_alias_05068 WHERE __aliasMarker(x + 1, computed) > 2 +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, computed) FROM t_dist_only_alias_05068 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; +SELECT 'alias_only_on_distributed_marker_off'; +SELECT count() FROM t_dist_only_alias_05068 WHERE __aliasMarker(x + 1, computed) > 2 +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; +SELECT __aliasMarker(x + 1, computed) FROM t_dist_only_alias_05068 ORDER BY x +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +SELECT 'pending_token_survives_scalar_rewrite'; +SELECT computed FROM t_dist_only_alias_05068 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, optimize_const_name_size = 0; + +CREATE TABLE t_right_05068 (x UInt64, extra UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_right_05068 VALUES (1, 10), (2, 20), (3, 30), (4, 40), (5, 50); + +-- `extra` is mentioned only in the marker's ID. It must still appear in the materialized right side. +SELECT 'global_join_id_only'; +SELECT l.x, __aliasMarker(1, r.extra) AS marker_value +FROM t_dist_only_alias_05068 AS l +GLOBAL INNER JOIN t_right_05068 AS r ON l.x = r.x +ORDER BY l.x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +DROP TABLE t_right_05068; +DROP TABLE t_dist_only_alias_05068; +DROP TABLE t_left_05068; +DROP TABLE t_dist_05068; +DROP TABLE t_local_05068; diff --git a/tests/queries/0_stateless/05069_alias_marker_projection_join_using.reference b/tests/queries/0_stateless/05069_alias_marker_projection_join_using.reference new file mode 100644 index 000000000000..d52367ba8326 --- /dev/null +++ b/tests/queries/0_stateless/05069_alias_marker_projection_join_using.reference @@ -0,0 +1,12 @@ +local +2 20 +3 30 +4 40 +distributed_marker_on +2 20 +3 30 +4 40 +distributed_marker_off +2 20 +3 30 +4 40 diff --git a/tests/queries/0_stateless/05069_alias_marker_projection_join_using.sql b/tests/queries/0_stateless/05069_alias_marker_projection_join_using.sql new file mode 100644 index 000000000000..1361c00b304c --- /dev/null +++ b/tests/queries/0_stateless/05069_alias_marker_projection_join_using.sql @@ -0,0 +1,32 @@ +-- `JOIN USING` can resolve a projection alias to an expression-bearing `ColumnNode`. +-- The matching query-tree test checks the injected side marker: `USING (k)` renders only the key name. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_left_05069; +DROP TABLE IF EXISTS t_dist_05069; +DROP TABLE IF EXISTS t_right_05069; + +CREATE TABLE t_left_05069 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_left_05069 VALUES (1), (2), (3); +CREATE TABLE t_dist_05069 AS t_left_05069 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_left_05069); +CREATE TABLE t_right_05069 (k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_right_05069 VALUES (2, 20), (3, 30), (4, 40); + +SELECT 'local'; +SELECT l.x + 1 AS k, r.v FROM t_left_05069 AS l INNER JOIN t_right_05069 AS r USING (k) ORDER BY k +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; + +SELECT 'distributed_marker_on'; +SELECT l.x + 1 AS k, r.v FROM t_dist_05069 AS l INNER JOIN t_right_05069 AS r USING (k) ORDER BY k +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1, enable_alias_marker = 1, + prefer_global_in_and_join = 1, prefer_localhost_replica = 0; + +SELECT 'distributed_marker_off'; +SELECT l.x + 1 AS k, r.v FROM t_dist_05069 AS l INNER JOIN t_right_05069 AS r USING (k) ORDER BY k +SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1, enable_alias_marker = 0, + prefer_global_in_and_join = 1, prefer_localhost_replica = 0; + +DROP TABLE t_right_05069; +DROP TABLE t_dist_05069; +DROP TABLE t_left_05069; diff --git a/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.reference b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.reference new file mode 100644 index 000000000000..18cb72eab2b4 --- /dev/null +++ b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.reference @@ -0,0 +1,8 @@ +local_pending +1 +1 +bad_token +bad_arity +distributed_sql +distributed_plan +global_in_subquery diff --git a/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql new file mode 100644 index 000000000000..b10810e31f7b --- /dev/null +++ b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql @@ -0,0 +1,36 @@ +-- The three-argument form is an internal pending marker. A local query evaluates its payload. +-- The distributed cases fail in the finalizer, since `1` is not a column id; `gtest_alias_marker_pending` covers the shipping check. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_local_05070; +DROP TABLE IF EXISTS t_dist_05070; +CREATE TABLE t_local_05070 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_05070 VALUES (1), (2); +CREATE TABLE t_dist_05070 AS t_local_05070 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05070); + +SELECT 'local_pending'; +SELECT __aliasMarker(1, 1, '__aliasMarker_pending_v1') FROM t_local_05070 ORDER BY x; + +SELECT 'bad_token'; +SELECT __aliasMarker(1, 1, 'wrong_token') FROM t_local_05070; -- { serverError BAD_ARGUMENTS } +SELECT __aliasMarker(NULL, 1, 'wrong_token') FROM t_local_05070; -- { serverError BAD_ARGUMENTS } + +SELECT 'bad_arity'; +SELECT __aliasMarker(1, 1, 'wrong_token', 1) FROM t_local_05070; -- { serverError BAD_ARGUMENTS } + +SELECT 'distributed_sql'; +SELECT count() FROM t_dist_05070 WHERE __aliasMarker(x + 1, 1, '__aliasMarker_pending_v1') > 2 +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0; -- { serverError BAD_ARGUMENTS } + +SELECT 'distributed_plan'; +SELECT count() FROM t_dist_05070 WHERE __aliasMarker(x + 1, 1, '__aliasMarker_pending_v1') > 2 +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; -- { serverError BAD_ARGUMENTS } + +SELECT 'global_in_subquery'; +SELECT count() FROM t_dist_05070 WHERE x GLOBAL IN + (SELECT __aliasMarker(number + 1, 1, '__aliasMarker_pending_v1') FROM numbers(1)) +SETTINGS prefer_localhost_replica = 0; -- { serverError BAD_ARGUMENTS } + +DROP TABLE t_dist_05070; +DROP TABLE t_local_05070; diff --git a/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.reference b/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.reference new file mode 100644 index 000000000000..fbf003931474 --- /dev/null +++ b/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.reference @@ -0,0 +1,4 @@ +sql_transport +1 1 +plan_attached_sql +1 1 diff --git a/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.sql b/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.sql new file mode 100644 index 000000000000..4ce3ae45f467 --- /dev/null +++ b/tests/queries/0_stateless/05071_alias_marker_shipped_sql_form.sql @@ -0,0 +1,48 @@ +-- The SQL attached to each remote query keeps the finalized two-argument marker, even when +-- the server also sends a serialized plan. The plan packet itself contains actions, not this SQL call. +SET enable_analyzer = 1; + +DROP TABLE IF EXISTS t_local_05071; +DROP TABLE IF EXISTS t_dist_05071; + +CREATE TABLE t_local_05071 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_05071 VALUES (1), (2), (3); +CREATE TABLE t_dist_05071 (x UInt64, computed UInt64 ALIAS x * 10) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05071); + +SELECT computed FROM t_dist_05071 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0, + log_queries = 1, log_comment = '05071_alias_marker_sql' FORMAT Null; + +SELECT computed FROM t_dist_05071 ORDER BY x +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1, + distributed_group_by_no_merge = 0, log_queries = 1, log_comment = '05071_alias_marker_plan' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +SELECT 'sql_transport'; +SELECT count() > 0 AS shipped, countIf( + position(query, '__aliasMarker(') > 0 + AND match(query, '__aliasMarker[(].+, ''__table[0-9]+[.]computed''[)]') + AND position(query, '__aliasMarker_pending_v1') = 0) = count() AS finalized +FROM system.query_log +WHERE type = 'QueryFinish' AND is_initial_query = 0 + AND initial_query_id IN ( + SELECT query_id FROM system.query_log + WHERE type = 'QueryFinish' AND is_initial_query = 1 + AND current_database = currentDatabase() AND log_comment = '05071_alias_marker_sql'); + +SELECT 'plan_attached_sql'; +SELECT count() > 0 AS shipped, countIf( + position(query, '__aliasMarker(') > 0 + AND match(query, '__aliasMarker[(].+, ''__table[0-9]+[.]computed''[)]') + AND position(query, '__aliasMarker_pending_v1') = 0) = count() AS finalized +FROM system.query_log +WHERE type = 'QueryFinish' AND is_initial_query = 0 + AND initial_query_id IN ( + SELECT query_id FROM system.query_log + WHERE type = 'QueryFinish' AND is_initial_query = 1 + AND current_database = currentDatabase() AND log_comment = '05071_alias_marker_plan'); + +DROP TABLE t_dist_05071; +DROP TABLE t_local_05071; From 1d7a7de291c8549f3d473dc30b1cede3565b7b5c Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 26 Sep 2026 14:27:15 +0200 Subject: [PATCH 12/18] tests: assert parallel-replica execution in `03931` The old `03931` passed even when parallel replicas were skipped. Use fail-loud settings for non-replicated `MergeTree` reads, correlate the `Distributed` AST query with secondary-query profile events, and add nested queries that assert the `buildQueryPlanForParallelReplicas` reconciliation step and replica participation in both SQL and serialized-plan transport. Keep the serialized `Distributed` plan case as a transport-only control; it does not use parallel replicas. Share settings across related queries to keep `EXPLAIN` and execution aligned. Test: `03931_parallel_replicas_alias_swap` passed with no random settings. --- ...931_parallel_replicas_alias_swap.reference | 13 ++- .../03931_parallel_replicas_alias_swap.sql | 79 +++++++++++++++---- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference index 14f9c770f714..3efdf9330f4c 100644 --- a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference @@ -1,9 +1,20 @@ local 10 11 12 20 21 22 +dist_plan +10 11 12 +20 21 22 pr_ast 10 11 12 20 21 22 -pr_plan +pr_query_ast +1 1 +10 11 12 +20 21 22 +pr_query_plan +1 1 10 11 12 20 21 22 +03931_dist_ast 1 1 +03931_query_ast 1 1 +03931_query_plan 1 1 diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql index f669631889c2..440320d4a4cc 100644 --- a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql @@ -1,11 +1,11 @@ --- Plain Distributed + parallel replicas (no Hybrid). Exercises the findParallelReplicasQuery --- header reconciliation path with nested ALIAS columns. Correct result equals the single-node --- ('local') result for both AST and serialized-plan transport. +-- The `Distributed` queries check nested `ALIAS` columns in AST and serialized-plan transport. +-- The nested queries over `MergeTree` exercise `buildQueryPlanForParallelReplicas` and its +-- `Convert distributed names` header reconciliation in both transports. A serialized +-- `Distributed` plan is built before the shard can enable parallel replicas. -- --- Determinism note: parallel replicas over a small non-replicated table can read the same rows on --- several replicas under some (randomized) settings, duplicating output. GROUP BY x, a1, a2 --- deduplicates that and keeps x in the required columns for the ALIAS expansion; ORDER BY x over --- distinct values gives a total order. The test still fails if a1/a2 are swapped or wrong. +-- Parallel replicas over a small non-replicated table can read the same rows on several replicas. +-- `GROUP BY x, a1, a2` deduplicates them and keeps x required for the `ALIAS` expansion; +-- `ORDER BY x` over distinct values gives a total order. Swapped or wrong aliases still fail. DROP TABLE IF EXISTS t_local_03931; DROP TABLE IF EXISTS t_dist_03931; @@ -19,19 +19,64 @@ ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDat SELECT 'local'; SELECT x, a1, a2 FROM t_local_03931 GROUP BY x, a1, a2 ORDER BY x; -SELECT 'pr_ast'; +SET enable_analyzer = 1, enable_alias_marker = 1; +SELECT 'dist_plan'; SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 1, - allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - serialize_query_plan = 0; +SETTINGS serialize_query_plan = 1; + +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_min_number_of_rows_per_replica = 0, + automatic_parallel_replicas_mode = 0, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 0; -SELECT 'pr_plan'; +SELECT 'pr_ast'; SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 1, - allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - serialize_query_plan = 1; +SETTINGS log_comment = '03931_dist_ast'; + +-- The subquery makes the query-level parallel-replica candidate eligible. Check the +-- distinctive reconciliation and remote-reading steps, not just the resulting values. +SELECT 'pr_query_ast'; +SELECT countIf(explain LIKE '%Convert distributed names%') > 0, + countIf(explain LIKE '%ReadFromRemoteParallelReplicas%') > 0 +FROM (EXPLAIN PLAN SELECT x, a1, a2 FROM (SELECT x, a1, a2 FROM t_local_03931) + GROUP BY x, a1, a2 ORDER BY x); +SELECT x, a1, a2 FROM (SELECT x, a1, a2 FROM t_local_03931) +GROUP BY x, a1, a2 ORDER BY x +SETTINGS log_comment = '03931_query_ast'; + +SET serialize_query_plan = 1; +SELECT 'pr_query_plan'; +SELECT countIf(explain LIKE '%Convert distributed names%') > 0, + countIf(explain LIKE '%ReadFromRemoteParallelReplicas%') > 0 +FROM (EXPLAIN PLAN SELECT x, a1, a2 FROM (SELECT x, a1, a2 FROM t_local_03931) + GROUP BY x, a1, a2 ORDER BY x); +SELECT x, a1, a2 FROM (SELECT x, a1, a2 FROM t_local_03931) +GROUP BY x, a1, a2 ORDER BY x +SETTINGS log_comment = '03931_query_plan'; + +SET allow_experimental_parallel_reading_from_replicas = 0, serialize_query_plan = 0; +SYSTEM FLUSH LOGS query_log; +-- The `Distributed` initiator does not itself count the replica reads. Correlate its +-- `initial_query_id` with secondary queries instead of checking only the initial row. +SELECT '03931_dist_ast', countIf(ProfileEvents['ParallelReplicasQueryCount'] > 0) > 0, + countIf(ProfileEvents['ParallelReplicasUsedCount'] > 0) > 0 +FROM system.query_log +WHERE event_date >= yesterday() AND type = 'QueryFinish' + AND initial_query_id IN + ( + SELECT query_id FROM system.query_log + WHERE event_date >= yesterday() AND type = 'QueryFinish' AND is_initial_query + AND current_database = currentDatabase() AND log_comment = '03931_dist_ast' + ); +SELECT log_comment, ProfileEvents['ParallelReplicasQueryCount'] > 0, + ProfileEvents['ParallelReplicasUsedCount'] > 0 +FROM system.query_log +WHERE event_date >= yesterday() AND type = 'QueryFinish' + AND is_initial_query AND current_database = currentDatabase() + AND log_comment IN ('03931_query_ast', '03931_query_plan') +ORDER BY log_comment; DROP TABLE t_dist_03931; DROP TABLE t_local_03931; From bcea758ebb88784d979d74e1ae8024a82958fc32 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 26 Sep 2026 15:18:57 +0200 Subject: [PATCH 13/18] Fix quoted `Merge` alias identifiers and row-policy inputs Match qualified child inputs using the planner quoting rule and emit requested `ALIAS` values under their canonical identifier. This prevents `Merge` over `Distributed` from returning defaults for quoted column names. Expose qualified child columns under plain names while evaluating a row policy, then project the filtered stream back to its original header before positional reconciliation. Add `05072` coverage for AST and serialized-plan reads, policies, intermediate aliases, and missing child columns. Tests: `05072` and the adjacent `Merge` alias, row-policy, and missing-column regressions passed with random settings disabled. --- src/Storages/StorageMerge.cpp | 80 ++++++++++++----- src/Storages/StorageMerge.h | 2 +- ...e_merge_quoted_alias_identifiers.reference | 29 +++++++ ...storage_merge_quoted_alias_identifiers.sql | 87 +++++++++++++++++++ 4 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.reference create mode 100644 tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.sql diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 21a897231afd..ac31e2db6ef2 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1456,10 +1456,49 @@ void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) step->addFilter(actions_dag.clone(), filter_column_name); } -void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const +void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan, const String & table_alias) const { + const auto original_header = plan.getCurrentHeader(); + const auto & header = *original_header; + ActionsDAG naming_actions(header.getColumnsWithTypeAndName()); + bool added_aliases = false; + /// The policy was analyzed against the child's plain column names. Expose the same values + /// under those names when the analyzer qualified them in the child stream. + for (const auto & column_name : filter_actions->getRequiredColumns()) + { + if (header.has(column_name)) + continue; + + const String identifier = table_alias.empty() ? column_name : backQuoteIfNeed(table_alias) + "." + backQuoteIfNeed(column_name); + if (!header.has(identifier)) + continue; + + naming_actions.addOrReplaceInOutputs(naming_actions.addAlias(naming_actions.findInOutputs(identifier), column_name)); + added_aliases = true; + } + + if (added_aliases) + { + auto naming_step = std::make_unique(plan.getCurrentHeader(), std::move(naming_actions)); + plan.addStep(std::move(naming_step)); + } + auto filter_step = std::make_unique(plan.getCurrentHeader(), actions_dag.clone(), filter_column_name, true /* remove filter column */); plan.addStep(std::move(filter_step)); + + if (added_aliases) + { + /// Do not let policy-only names change the column count or order used for the later + /// positional reconciliation with the Merge table's expected header. + ActionsDAG projection_actions(plan.getCurrentHeader()->getColumnsWithTypeAndName()); + NamesWithAliases original_names; + original_names.reserve(header.columns()); + for (const auto & column : header) + original_names.emplace_back(column.name, ""); + projection_actions.project(original_names); + auto projection_step = std::make_unique(plan.getCurrentHeader(), std::move(projection_actions)); + plan.addStep(std::move(projection_step)); + } } StorageMerge::StorageListWithLocks ReadFromMerge::getSelectedTables( @@ -1688,12 +1727,12 @@ void ReadFromMerge::convertAndFilterSourceStream( /// /// The planner's `TableExpressionData` for the Merge node is not populated yet at this point -- column /// collection happens later, in `CollectTableExpressionData` -- so the mapping cannot be looked up through - /// `PlannerContext`. Build it here instead, by matching each Merge-declared column name against the suffixes of - /// the identifier names in `header` / `pipe_columns`: a column named `.` or ``.`` `` - /// belongs to the Merge column `C`. + /// `PlannerContext`. For inputs, match each Merge-declared column name against the suffixes of + /// the identifier names in `pipe_columns`: a column named + /// `.` belongs to the Merge column `C`. /// - /// Driving the match from the declared Merge schema rather than from a pattern over the analyzer's naming - /// convention is what makes dotted names work, `Nested` subcolumns and backtick-quoted names included. + /// Matching declared Merge names with the planner's quoting rule also handles dotted names, + /// `Nested` subcolumns, and names requiring backticks. /// `getAll` builds a fresh list on every call, so take it once rather than per candidate column. const auto all_merge_columns = merge_columns.getAll(); @@ -1712,18 +1751,12 @@ void ReadFromMerge::convertAndFilterSourceStream( continue; } - /// Otherwise look for `.` or ``.`` `` where C is a declared Merge column. Columns - /// matching no declared name are skipped; they are intermediate outputs of the child plan. - /// - /// The unquoted form of a dotted name is accepted defensively. No producer of it is known: - /// `buildColumnIdentifier` always backquotes a dotted column name, so an analyzer identifier for `n.a` - /// is always ``__tableN.`n.a` ``. + /// Otherwise match the quoted identifier component emitted by `buildColumnIdentifier`. + /// Columns matching no declared name are intermediate outputs of the child plan. for (const auto & merge_column : all_merge_columns) { - const bool dotted = merge_column.name.find('.') != String::npos; - const String want_raw = "." + merge_column.name; - const String want_quoted = dotted ? ("." + backQuote(merge_column.name)) : want_raw; - if (column.name.ends_with(want_quoted) || (dotted && column.name.ends_with(want_raw))) + const String qualified_suffix = "." + backQuoteIfNeed(merge_column.name); + if (column.name.ends_with(qualified_suffix)) { if (!plain_to_identifier.emplace(merge_column.name, column.name).second) ambiguous.insert(merge_column.name); @@ -1736,7 +1769,6 @@ void ReadFromMerge::convertAndFilterSourceStream( return plain_to_identifier; }; - const auto header_plain_to_identifier = build_plain_to_identifier(header); const auto pipe_plain_to_identifier = build_plain_to_identifier(pipe_columns); for (const auto & alias : aliases) @@ -1782,11 +1814,12 @@ void ReadFromMerge::convertAndFilterSourceStream( if (nodes.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size()); - /// Emit the alias output under its analyzer identifier, so the `addMissingDefaults` step further down - /// matches it by name. Emitted under the plain name it would miss, and the expected - /// ``__tableN.`alias.name` `` column would be filled with type defaults instead. - const auto identifier_it = header_plain_to_identifier.find(alias.name); - const String & output_name = identifier_it != header_plain_to_identifier.end() ? identifier_it->second : alias.name; + /// Emit an alias needed by the output header under its analyzer identifier. An alias + /// needed only as an input to another alias keeps its plain logical name. + const auto & source_alias = modified_query_info.table_expression->getAlias(); + const String expected_identifier = source_alias.empty() + ? alias.name : backQuoteIfNeed(source_alias) + "." + backQuoteIfNeed(alias.name); + const String & output_name = header.has(expected_identifier) ? expected_identifier : alias.name; actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), output_name)); auto expression_step = std::make_unique(child.plan.getCurrentHeader(), std::move(actions_dag)); child.plan.addStep(std::move(expression_step)); @@ -1810,7 +1843,8 @@ void ReadFromMerge::convertAndFilterSourceStream( /// This is the filter for the individual source table, that's why filtering has to be done before all structure adaptations. if (row_policy_data_opt) - row_policy_data_opt->addFilterTransform(child.plan); + row_policy_data_opt->addFilterTransform( + child.plan, modified_query_info.table_expression ? modified_query_info.table_expression->getAlias() : ""); /** Output headers may differ from what StorageMerge expects in some cases. * When the child table engine produces a query plan for the stage after FetchColumns, diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 34efb3fd5e7e..63698f8bbb5d 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -258,7 +258,7 @@ class ReadFromMerge final : public SourceStepWithFilter /// Create explicit filter transform to exclude /// rows that are not conform to row level policy - void addFilterTransform(QueryPlan &) const; + void addFilterTransform(QueryPlan &, const String & table_alias) const; private: std::string filter_column_name; // complex filter, may contain logic operations diff --git a/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.reference b/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.reference new file mode 100644 index 000000000000..ae525de7485f --- /dev/null +++ b/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.reference @@ -0,0 +1,29 @@ +local source +1 10 100 11 +2 20 200 21 +local Merge +1 10 100 11 +2 20 200 21 +Distributed AST +1 10 100 +2 20 200 +Distributed plan +1 10 100 +2 20 200 +intermediate alias +1 11 +2 21 +row policy +2 20 +row policy all aliases +2 20 200 +policy-only alias +2 +flag alias policy +2 1 +flag alias policy-only column +2 +missing child aliases +1 10 100 11 +2 20 200 21 +3 0 0 0 diff --git a/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.sql b/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.sql new file mode 100644 index 000000000000..afed9851d6e8 --- /dev/null +++ b/tests/queries/0_stateless/05072_storage_merge_quoted_alias_identifiers.sql @@ -0,0 +1,87 @@ +-- Qualified `ALIAS` names must use the planner's quoting convention in `Merge` header +-- reconciliation. A missing match used to fill both quoted outputs with zeros. +SET enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE IF EXISTS t_merge_dist_05072; +DROP TABLE IF EXISTS t_merge_local_05072; +DROP TABLE IF EXISTS t_dist_05072; +DROP TABLE IF EXISTS t_local_05072; + +CREATE TABLE t_local_05072 +( + id UInt32, + `a-b` UInt32 ALIAS id * 10, + `a b` UInt32 ALIAS id * 100, + chained UInt32 ALIAS `a-b` + 1, + flag UInt8 ALIAS id = 2 +) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_local_05072 VALUES (1), (2); + +CREATE TABLE t_dist_05072 AS t_local_05072 +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_local_05072); +CREATE TABLE t_merge_local_05072 +(id UInt32, `a-b` UInt32, `a b` UInt32, chained UInt32, flag UInt8) +ENGINE = Merge(currentDatabase(), '^t_local_05072$'); +CREATE TABLE t_merge_dist_05072 AS t_merge_local_05072 +ENGINE = Merge(currentDatabase(), '^t_dist_05072$'); + +SELECT 'local source'; +SELECT id, `a-b`, `a b`, chained FROM t_local_05072 ORDER BY id; + +SELECT 'local Merge'; +SELECT id, `a-b`, `a b`, chained FROM t_merge_local_05072 ORDER BY id; + +SELECT 'Distributed AST'; +SELECT id, `a-b`, `a b` FROM t_merge_dist_05072 +GROUP BY id, `a-b`, `a b` ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0; + +SELECT 'Distributed plan'; +SELECT id, `a-b`, `a b` FROM t_merge_dist_05072 +GROUP BY id, `a-b`, `a b` ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; + +-- The `a-b` alias is used to calculate `chained`, but is not an output column. +SELECT 'intermediate alias'; +SELECT id, chained FROM t_merge_dist_05072 GROUP BY id, chained ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +-- The row policy on the `Distributed` child runs after the qualified alias reconstruction. +CREATE ROW POLICY rp_05072 ON t_dist_05072 USING id >= 2 AND `a-b` >= 20 TO ALL; +SELECT 'row policy'; +SELECT id, `a-b` FROM t_merge_dist_05072 GROUP BY id, `a-b` ORDER BY id +SETTINGS prefer_localhost_replica = 0; +SELECT 'row policy all aliases'; +SELECT id, `a-b`, `a b` FROM t_merge_dist_05072 +GROUP BY id, `a-b`, `a b` ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; +SELECT 'policy-only alias'; +SELECT id FROM t_merge_dist_05072 GROUP BY id ORDER BY id +SETTINGS prefer_localhost_replica = 0; +DROP ROW POLICY rp_05072 ON t_dist_05072; + +-- A policy over a UInt8 `ALIAS` preserves the selected column and also works when +-- the alias is needed only for the filter. +CREATE ROW POLICY rp_05072 ON t_dist_05072 USING flag = 1 TO ALL; +SELECT 'flag alias policy'; +SELECT id, flag FROM t_merge_dist_05072 GROUP BY id, flag ORDER BY id +SETTINGS prefer_localhost_replica = 0; +SELECT 'flag alias policy-only column'; +SELECT id FROM t_merge_dist_05072 GROUP BY id ORDER BY id +SETTINGS prefer_localhost_replica = 0; +DROP ROW POLICY rp_05072 ON t_dist_05072; + +-- A child without these columns still uses the ordinary missing-column defaults. +CREATE TABLE t_missing_05072 (id UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_missing_05072 VALUES (3); +CREATE TABLE t_merge_mixed_05072 AS t_merge_local_05072 +ENGINE = Merge(currentDatabase(), '^t_(local|missing)_05072$'); +SELECT 'missing child aliases'; +SELECT id, `a-b`, `a b`, chained FROM t_merge_mixed_05072 ORDER BY id; +DROP TABLE t_merge_mixed_05072; +DROP TABLE t_missing_05072; + +DROP TABLE t_merge_dist_05072; +DROP TABLE t_merge_local_05072; +DROP TABLE t_dist_05072; +DROP TABLE t_local_05072; From 30933507a65e93239011afdd5f31a9e1ec3d4e02 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Sat, 26 Sep 2026 15:45:43 +0200 Subject: [PATCH 14/18] Preserve bare row-policy inputs in `StorageMerge` A bare `USING flag` policy reuses its input column rather than adding a computed filter result. Recognize that input in `RowPolicyData` and keep it after the explicit `FilterStep`; retain the existing removal behavior for computed policy results. Add `05073` covering bare physical and `ALIAS` policies through local `Merge` and `Merge` over `Distributed`, including non-Boolean truthy values and serialized-plan transport. Before the fix, the test raised a `LOGICAL_ERROR`; after the fix, `05073`, `05072`, `02763`, and `04305` pass. --- src/Storages/StorageMerge.cpp | 17 +++-- src/Storages/StorageMerge.h | 1 + ...age_merge_bare_row_policy_column.reference | 24 +++++++ ...3_storage_merge_bare_row_policy_column.sql | 64 +++++++++++++++++++ 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.reference create mode 100644 tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.sql diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index ac31e2db6ef2..9a57de12e3c3 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1422,14 +1422,23 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter NamesAndTypesList added; NamesAndTypesList deleted; sample_block_columns.getDifference(required_columns, added, deleted); - if (!deleted.empty() || added.size() != 1) + if (deleted.empty() && added.empty() && expr->as() + && actions_dag.findInOutputs(expr->getColumnName()).type == ActionsDAG::ActionType::INPUT) + { + /// A bare column policy filters on an existing input, which must remain available after filtering. + filter_column_name = expr->getColumnName(); + remove_filter_column = false; + } + else if (deleted.empty() && added.size() == 1) + { + filter_column_name = added.getNames().front(); + } + else { throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot determine row level filter; {} columns deleted, {} columns added", deleted.size(), added.size()); } - - filter_column_name = added.getNames().front(); } void ReadFromMerge::RowPolicyData::extendNames(Names & names) const @@ -1483,7 +1492,7 @@ void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan, const St plan.addStep(std::move(naming_step)); } - auto filter_step = std::make_unique(plan.getCurrentHeader(), actions_dag.clone(), filter_column_name, true /* remove filter column */); + auto filter_step = std::make_unique(plan.getCurrentHeader(), actions_dag.clone(), filter_column_name, remove_filter_column); plan.addStep(std::move(filter_step)); if (added_aliases) diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 63698f8bbb5d..7d08bea8e7c1 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -262,6 +262,7 @@ class ReadFromMerge final : public SourceStepWithFilter private: std::string filter_column_name; // complex filter, may contain logic operations + bool remove_filter_column = true; ActionsDAG actions_dag; ExpressionActionsPtr filter_actions; StorageMetadataPtr storage_metadata_snapshot; diff --git a/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.reference b/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.reference new file mode 100644 index 000000000000..10652999a625 --- /dev/null +++ b/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.reference @@ -0,0 +1,24 @@ +local alias selected +2 5 +local alias policy-only +2 +local physical selected +2 1 +3 1 +4 5 +local physical policy-only +2 +3 +4 +Distributed AST alias selected +2 5 +Distributed plan alias selected +2 5 +Distributed alias policy-only +2 +Distributed plan alias policy-only +2 +Distributed physical selected +2 1 +3 1 +4 5 diff --git a/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.sql b/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.sql new file mode 100644 index 000000000000..87095e28b05a --- /dev/null +++ b/tests/queries/0_stateless/05073_storage_merge_bare_row_policy_column.sql @@ -0,0 +1,64 @@ +-- A bare row-policy column is the filter input, not a newly computed output. +-- The value must survive filtering when selected, and rows with a false value +-- must be excluded even when the column is used only by the policy. +SET enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE IF EXISTS t_merge_dist_05073; +DROP TABLE IF EXISTS t_merge_local_05073; +DROP TABLE IF EXISTS t_dist_05073; +DROP TABLE IF EXISTS t_local_05073; + +CREATE TABLE t_local_05073 +( + id UInt32, + visible UInt8, + flag UInt8 ALIAS if(id = 2, 5, 0) +) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_local_05073 VALUES (1, 0), (2, 1), (3, 1), (4, 5); + +CREATE TABLE t_dist_05073 AS t_local_05073 +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_local_05073); +CREATE TABLE t_merge_local_05073 (id UInt32, visible UInt8, flag UInt8) +ENGINE = Merge(currentDatabase(), '^t_local_05073$'); +CREATE TABLE t_merge_dist_05073 AS t_merge_local_05073 +ENGINE = Merge(currentDatabase(), '^t_dist_05073$'); + +CREATE ROW POLICY rp_05073 ON t_local_05073 USING flag TO ALL; +SELECT 'local alias selected'; +SELECT id, flag FROM t_merge_local_05073 ORDER BY id; +SELECT 'local alias policy-only'; +SELECT id FROM t_merge_local_05073 ORDER BY id; +DROP ROW POLICY rp_05073 ON t_local_05073; + +CREATE ROW POLICY rp_05073 ON t_local_05073 USING visible TO ALL; +SELECT 'local physical selected'; +SELECT id, visible FROM t_merge_local_05073 ORDER BY id; +SELECT 'local physical policy-only'; +SELECT id FROM t_merge_local_05073 ORDER BY id; +DROP ROW POLICY rp_05073 ON t_local_05073; + +CREATE ROW POLICY rp_05073 ON t_dist_05073 USING flag TO ALL; +SELECT 'Distributed AST alias selected'; +SELECT id, flag FROM t_merge_dist_05073 GROUP BY id, flag ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0; +SELECT 'Distributed plan alias selected'; +SELECT id, flag FROM t_merge_dist_05073 GROUP BY id, flag ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; +SELECT 'Distributed alias policy-only'; +SELECT id FROM t_merge_dist_05073 GROUP BY id ORDER BY id +SETTINGS prefer_localhost_replica = 0; +SELECT 'Distributed plan alias policy-only'; +SELECT id FROM t_merge_dist_05073 GROUP BY id ORDER BY id +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; +DROP ROW POLICY rp_05073 ON t_dist_05073; + +CREATE ROW POLICY rp_05073 ON t_dist_05073 USING visible TO ALL; +SELECT 'Distributed physical selected'; +SELECT id, visible FROM t_merge_dist_05073 GROUP BY id, visible ORDER BY id +SETTINGS prefer_localhost_replica = 0; +DROP ROW POLICY rp_05073 ON t_dist_05073; + +DROP TABLE t_merge_dist_05073; +DROP TABLE t_merge_local_05073; +DROP TABLE t_dist_05073; +DROP TABLE t_local_05073; From 685d5d3de4854f45284d001a205c76245ab543d4 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 29 Sep 2026 07:46:30 +0200 Subject: [PATCH 15/18] Consolidate alias-marker regression coverage Fold overlapping `Distributed` and `Merge` alias tests into focused fixtures, and remove unrelated `Hybrid` unknown-table coverage from this branch. Keep `05057` rather than its duplicate `04281`. Strengthen assertions for nonempty `ALIAS` values, meaningful `LIMIT`, and actual parallel-replica execution. Rename the remote-over-`Distributed` second-hop test after verifying that its old parallel-replica path was not used. The twelve affected and adjacent stateless tests passed. --- ...lias_marker_with_mergeable_state.reference | 2 - ...03648_alias_marker_with_mergeable_state.sh | 9 - .../03842_hybrid_alias_issue_1424.reference | 11 - .../03842_hybrid_alias_issue_1424.sql | 17 +- ...replicas_second_hop_alias_marker.reference | 4 - ...te_over_distributed_alias_marker.reference | 6 + ..._remote_over_distributed_alias_marker.sql} | 25 +- ...32_distributed_alias_strict_name.reference | 9 - .../03932_distributed_alias_strict_name.sql | 27 -- ...uted_alias_marker_setting_effect.reference | 28 -- ...istributed_alias_marker_setting_effect.sql | 42 --- ...age_merge_over_distributed_alias.reference | 20 -- ...1_storage_merge_over_distributed_alias.sql | 58 --- ...istributed_alias_same_expression.reference | 14 +- ...5053_distributed_alias_same_expression.sql | 2 +- ...nown_table_issues_1208_1209_1422.reference | 8 - ...id_unknown_table_issues_1208_1209_1422.sql | 118 ------ ...8_distributed_alias_swap_planner.reference | 9 + .../05058_distributed_alias_swap_planner.sql | 42 ++- ...ted_alias_merge_over_distributed.reference | 9 - ...59_dotted_alias_merge_over_distributed.sql | 64 ---- ...ybrid_unknown_table_exact_schema.reference | 35 -- ...5060_hybrid_unknown_table_exact_schema.sql | 338 ------------------ ...alias_column_swap_without_marker.reference | 14 + ...buted_alias_column_swap_without_marker.sql | 9 + ...l_replicas_dod_alias_column_swap.reference | 3 + ...arallel_replicas_dod_alias_column_swap.sql | 29 +- ...ted_alias_merge_over_distributed.reference | 9 + ...ed_dotted_alias_merge_over_distributed.sql | 27 +- 29 files changed, 136 insertions(+), 852 deletions(-) delete mode 100644 tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference create mode 100644 tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.reference rename tests/queries/0_stateless/{03847_parallel_replicas_second_hop_alias_marker.sql => 03847_remote_over_distributed_alias_marker.sql} (55%) delete mode 100644 tests/queries/0_stateless/03932_distributed_alias_strict_name.reference delete mode 100644 tests/queries/0_stateless/03932_distributed_alias_strict_name.sql delete mode 100644 tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference delete mode 100644 tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql delete mode 100644 tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference delete mode 100644 tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql delete mode 100644 tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference delete mode 100644 tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql delete mode 100644 tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference delete mode 100644 tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql delete mode 100644 tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference delete mode 100644 tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference index 5f061a829b23..58bf6a7ec74b 100644 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference @@ -2,8 +2,6 @@ Header: sum(foo) AggregateFunction(sum, Int64) ---- stage: with_mergeable_state (analyzer=0) ---- Expected error: Function __aliasMarker is internal and supported only with the analyzer ----- explicit __aliasMarker in user query (analyzer=1) ---- -Explicit __aliasMarker call is allowed ---- stage: complete (analyzer=1) ---- Header: x Int64 ---- stage: fetch_columns (analyzer=1) ---- diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh index 1125a06b3d54..79409806c073 100755 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh @@ -27,15 +27,6 @@ else echo "${alias_marker_error_output}" fi -echo "---- explicit __aliasMarker in user query (analyzer=1) ----" -if $CLICKHOUSE_CLIENT --enable_analyzer=1 --query \ - "SELECT __aliasMarker(number*2-3,'foo') FROM numbers(1)" >/dev/null 2>&1 -then - echo "Explicit __aliasMarker call is allowed" -else - echo "Unexpected error for explicit __aliasMarker call" -fi - echo "---- stage: complete (analyzer=1) ----" $CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage complete --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference index 6f78da4c4f59..e6fabb5220b6 100644 --- a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference @@ -8,13 +8,6 @@ cte with limit -2147483648 -4294967296 -1762862292 -574613778 -1329695183 -1573638336 --221724287 679772422 -0 0 -550067609 -3048000734 -1084637461 3417479706 -1169291374 -3082049462 -1899628504 -740161250 -2147483647 4294967294 cte without limit -2147483648 -4294967296 -1762862292 -574613778 @@ -32,10 +25,6 @@ intersect with order by -221724287 679772422 1084637461 3417479706 2147483647 4294967294 -intersect without order by --221724287 679772422 -1084637461 3417479706 -2147483647 4294967294 constant alias in subquery 9 7 32 constant alias predicate diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql index 8b9cf9182896..27f29faa1943 100644 --- a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql @@ -85,7 +85,8 @@ WITH ranked AS ( SELECT id, computed FROM test_hybrid_issue_1424 - LIMIT 10 + ORDER BY id + LIMIT 3 ) SELECT * FROM ranked @@ -124,20 +125,6 @@ FROM ) ORDER BY id; -SELECT 'intersect without order by'; -SELECT * -FROM -( - SELECT id, computed - FROM test_hybrid_issue_1424 - WHERE computed > 100 - INTERSECT - SELECT id, computed - FROM test_hybrid_issue_1424 - WHERE value > 50 -) -ORDER BY id; - CREATE TABLE test_hybrid_issue_1424_const_left ( id Int32, diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference deleted file mode 100644 index fbdae0d35623..000000000000 --- a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference +++ /dev/null @@ -1,4 +0,0 @@ -single_replica_second_hop -1999-03-29 01:15:33.000 x x -parallel_replicas_second_hop -1999-03-29 01:15:33.000 x x diff --git a/tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.reference b/tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.reference new file mode 100644 index 000000000000..9ae0b7c40d76 --- /dev/null +++ b/tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.reference @@ -0,0 +1,6 @@ +ast_second_hop +1999-03-29 01:15:33.000 x x +1999-03-29 01:15:34.000 y y +plan_second_hop +1999-03-29 01:15:33.000 x x +1999-03-29 01:15:34.000 y y diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql b/tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.sql similarity index 55% rename from tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql rename to tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.sql index 5500c4904f82..eac350656a2e 100644 --- a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql +++ b/tests/queries/0_stateless/03847_remote_over_distributed_alias_marker.sql @@ -1,5 +1,4 @@ --- Regression coverage for materialized __aliasMarker metadata across --- remote -> Distributed -> parallel replicas fanout. +-- Regression coverage for nested `ALIAS` columns across `remote` -> `Distributed` -> `MergeTree`. DROP TABLE IF EXISTS test_alias_pr_second_hop_dist; DROP TABLE IF EXISTS test_alias_pr_second_hop_local; @@ -21,31 +20,17 @@ INSERT INTO test_alias_pr_second_hop_local VALUES CREATE TABLE test_alias_pr_second_hop_dist AS test_alias_pr_second_hop_local ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_alias_pr_second_hop_local); -SELECT 'single_replica_second_hop'; +SELECT 'ast_second_hop'; SELECT dt, alias_base_0, alias_base_1 FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) ORDER BY dt -LIMIT 1 -SETTINGS - enable_analyzer = 1, - enable_alias_marker = 1, - enable_parallel_replicas = 1, - max_parallel_replicas = 1, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - parallel_replicas_for_non_replicated_merge_tree = 1; +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 0; -SELECT 'parallel_replicas_second_hop'; +SELECT 'plan_second_hop'; SELECT dt, alias_base_0, alias_base_1 FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) ORDER BY dt -LIMIT 1 -SETTINGS - enable_analyzer = 1, - enable_alias_marker = 1, - enable_parallel_replicas = 1, - max_parallel_replicas = 3, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', - parallel_replicas_for_non_replicated_merge_tree = 1; +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1; DROP TABLE test_alias_pr_second_hop_dist; DROP TABLE test_alias_pr_second_hop_local; diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference b/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference deleted file mode 100644 index cddf594d4e31..000000000000 --- a/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference +++ /dev/null @@ -1,9 +0,0 @@ -local -12 11 23 -22 21 43 -dist -12 11 23 -22 21 43 -dist_plan -12 11 23 -22 21 43 diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql b/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql deleted file mode 100644 index c094d28f01e0..000000000000 --- a/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql +++ /dev/null @@ -1,27 +0,0 @@ --- Plain Distributed (no Hybrid). Reorders alias columns and mixes a computed expression over --- them. With strict name-based header reconciliation (positional fallback disabled), the result --- must equal the single-node ('local') result for both AST and serialized-plan transport, and no --- LOGICAL_ERROR must be raised. -DROP TABLE IF EXISTS t_local_03932; -DROP TABLE IF EXISTS t_dist_03932; - -CREATE TABLE t_local_03932 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) -ENGINE = MergeTree ORDER BY x; -INSERT INTO t_local_03932 VALUES (10), (20); - -CREATE TABLE t_dist_03932 AS t_local_03932 -ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03932); - -SELECT 'local'; -SELECT a2, a1, a1 + a2 AS s FROM t_local_03932 ORDER BY x; - -SELECT 'dist'; -SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; - -SELECT 'dist_plan'; -SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; - -DROP TABLE t_dist_03932; -DROP TABLE t_local_03932; diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference deleted file mode 100644 index d015dccfb64f..000000000000 --- a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference +++ /dev/null @@ -1,28 +0,0 @@ -marker_on -x a_str inner_c -1 aaaa 2 -1 aaaa 2 -1 aaaa 2 -1 aaaa 2 -2 aaaa 3 -2 aaaa 3 -2 aaaa 3 -2 aaaa 3 -10 aaaa 11 -10 aaaa 11 -10 aaaa 11 -10 aaaa 11 -marker_off -x a_str inner_c -1 aaaa 2 -1 aaaa 2 -1 aaaa 2 -1 aaaa 2 -2 aaaa 3 -2 aaaa 3 -2 aaaa 3 -2 aaaa 3 -10 aaaa 11 -10 aaaa 11 -10 aaaa 11 -10 aaaa 11 diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql deleted file mode 100644 index 648c98d7553b..000000000000 --- a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql +++ /dev/null @@ -1,42 +0,0 @@ --- `enable_alias_marker` must not change results. It exists so an initiator can stop emitting --- `__aliasMarker` for a mixed-version cluster whose shards do not understand it, which is a --- transport concern, not a correctness one. --- --- Distributed-over-distributed with a String ALIAS (`a_str`) and a UInt64 ALIAS (`inner_c`). This --- shape used to swap the two columns with the marker off, routing the String 'aaaa' into the --- UInt64 `inner_c` slot and failing with CANNOT_PARSE_TEXT. Upstream fixed the underlying column --- ordering, so both settings now return the same rows, and this test holds them to that. -DROP TABLE IF EXISTS t_se_local; -DROP TABLE IF EXISTS t_se_inner; -DROP TABLE IF EXISTS t_se_outer; - -CREATE TABLE t_se_local (x UInt64) ENGINE = MergeTree() ORDER BY x; -INSERT INTO t_se_local VALUES (1), (2), (10); - -CREATE TABLE t_se_inner (x UInt64, inner_c UInt64 ALIAS x + 1) -ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_local); - -CREATE TABLE t_se_outer (x UInt64, inner_c UInt64, a_str String ALIAS 'aaaa') -ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_inner); - --- serialize_query_plan is pinned to 0 throughout: this test targets the AST-path alias marker. --- On the serialized-plan path the header is reconciled by name regardless of the marker, so the --- marker_off swap below does not occur there; the "distributed plan" CI flavor would otherwise --- force the plan path on and the marker_off query would succeed instead of erroring. -SELECT 'marker_on'; -SELECT x, a_str, inner_c -FROM t_se_outer -ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0 -FORMAT TSVWithNames; - -SELECT 'marker_off'; -SELECT x, a_str, inner_c -FROM t_se_outer -ORDER BY x -SETTINGS enable_analyzer = 1, enable_alias_marker = 0, prefer_localhost_replica = 0, serialize_query_plan = 0 -FORMAT TSVWithNames; - -DROP TABLE t_se_outer; -DROP TABLE t_se_inner; -DROP TABLE t_se_local; diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference deleted file mode 100644 index f32381f38096..000000000000 --- a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference +++ /dev/null @@ -1,20 +0,0 @@ -local -1 2 3 -2 3 4 -10 11 12 -merge_prefer0 -1 2 3 -2 3 4 -10 11 12 -merge_prefer1 -1 2 3 -2 3 4 -10 11 12 -merge_prefer0_plan -1 2 3 -2 3 4 -10 11 12 -merge_prefer1_plan -1 2 3 -2 3 4 -10 11 12 diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql deleted file mode 100644 index e1ac59428d5e..000000000000 --- a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql +++ /dev/null @@ -1,58 +0,0 @@ --- Plain Merge over Distributed over MergeTree without an explicit __aliasMarker call. --- Nested ALIAS columns (b contains a's subexpression). Reading the alias columns through the --- Merge table must reconcile the child (Distributed) header by name; a positional reconciliation --- in StorageMerge::convertAndFilterSourceStream would swap the columns (or fill them with 0). --- The correct result equals the single-node ('local') result. --- --- Determinism notes: `x` is kept in GROUP BY so the ALIAS expansion can resolve it (the alias --- expressions are defined in terms of x); GROUP BY also deduplicates the rows the two shards --- produce, and ORDER BY x (distinct values) gives a total order independent of the distributed --- merge order. So every block - local and the distributed variants - yields the same rows. -DROP TABLE IF EXISTS test_merge_alias_swap_merge; -DROP TABLE IF EXISTS test_merge_alias_swap_dist; -DROP TABLE IF EXISTS test_merge_alias_swap_local; - -CREATE TABLE test_merge_alias_swap_local -( - x UInt64, - a UInt64 ALIAS x + 1, - b UInt64 ALIAS a + 1 -) -ENGINE = MergeTree() -ORDER BY x; - -INSERT INTO test_merge_alias_swap_local VALUES (1), (2), (10); - -CREATE TABLE test_merge_alias_swap_dist AS test_merge_alias_swap_local -ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_merge_alias_swap_local); - -CREATE TABLE test_merge_alias_swap_merge -( - x UInt64, - a UInt64, - b UInt64 -) -ENGINE = Merge(currentDatabase(), '^test_merge_alias_swap_dist$'); - -SELECT 'local'; -SELECT x, a, b FROM test_merge_alias_swap_local GROUP BY x, a, b ORDER BY x; - -SELECT 'merge_prefer0'; -SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x -SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0; - -SELECT 'merge_prefer1'; -SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x -SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1; - -SELECT 'merge_prefer0_plan'; -SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x -SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; - -SELECT 'merge_prefer1_plan'; -SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x -SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; - -DROP TABLE test_merge_alias_swap_merge; -DROP TABLE test_merge_alias_swap_dist; -DROP TABLE test_merge_alias_swap_local; diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference index 8ddb26d0b961..25d3e1f10881 100644 --- a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference @@ -1,16 +1,16 @@ first -1999-03-29 01:15:33.000 +1999-03-29 01:15:33.000 nonempty nonempty second -1999-03-29 01:15:33.000 +1999-03-29 01:15:33.000 nonempty nonempty third -1999-03-29 01:15:33.000 +1999-03-29 01:15:33.000 nonempty nonempty fourth -1999-03-29 01:15:33.000 +1999-03-29 01:15:33.000 nonempty nonempty fifth -1999-03-29 01:15:33.000 +1999-03-29 01:15:33.000 nonempty nonempty sixth query_alias_0 query_alias_1 - +nonempty nonempty seventh alias_String_7_0 alias_String_7_1 - +nonempty nonempty diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql index 20cf77930d0b..d6589354bd75 100644 --- a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql @@ -22,7 +22,7 @@ CREATE TABLE test_alias_same_expr_remote ENGINE = MergeTree() ORDER BY dt; -INSERT INTO test_alias_same_expr_remote VALUES ('1999-03-29T01:15:33', ''); +INSERT INTO test_alias_same_expr_remote VALUES ('1999-03-29T01:15:33', 'nonempty'); SELECT 'first'; SELECT dt, alias_String_7_0, alias_String_7_1 diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference deleted file mode 100644 index 5155d27310c2..000000000000 --- a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference +++ /dev/null @@ -1,8 +0,0 @@ -issue_1208_self_in_subquery -5 -issue_1209_join_mode_local -6 -issue_1209_join_mode_allow -6 -issue_1422_hybrid_in_merge_tree_subquery -5 diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql deleted file mode 100644 index c5cab11cee86..000000000000 --- a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql +++ /dev/null @@ -1,118 +0,0 @@ -SET allow_experimental_hybrid_table = 1, - enable_analyzer = 1, - prefer_localhost_replica = 0, - -- AST-path regression test for unknown-table issues #1208/#1209/#1422. Pin - -- serialize_query_plan=0 so the "distributed plan" CI flavor (which forces it on) does not - -- route these hybrid + IN-subquery queries through the plan path, which has a separate, - -- unrelated header-reconciliation gap. - serialize_query_plan = 0; - -DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422; -DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_left; -DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_right; -DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_mt; - -CREATE TABLE test_hybrid_issue_1208_1209_1422_left -( - string_col String, - long_col Int64, - date_col Date -) -ENGINE = MergeTree -ORDER BY string_col; - -CREATE TABLE test_hybrid_issue_1208_1209_1422_right -( - string_col String, - long_col Int64, - date_col Date -) -ENGINE = MergeTree -ORDER BY string_col; - -CREATE TABLE test_hybrid_issue_1208_1209_1422_mt -( - string_col String, - long_col Int64, - date_col Date -) -ENGINE = MergeTree -ORDER BY string_col; - -INSERT INTO test_hybrid_issue_1208_1209_1422_left VALUES - ('William', 9044, toDate('2024-01-01')), - ('Oliver', 1654, toDate('2024-01-01')), - ('Frank', 8751, toDate('2024-01-01')); - -INSERT INTO test_hybrid_issue_1208_1209_1422_right VALUES - ('Louis', 1519, toDate('2024-01-02')), - ('Isaac', 3611, toDate('2024-01-02')); - -INSERT INTO test_hybrid_issue_1208_1209_1422_mt -SELECT * FROM test_hybrid_issue_1208_1209_1422_left -UNION ALL -SELECT * FROM test_hybrid_issue_1208_1209_1422_right; - -CREATE TABLE test_hybrid_issue_1208_1209_1422 -( - string_col String, - long_col Int64, - date_col Date -) -ENGINE = Hybrid( - remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_left'), date_col <= '2024-01-01', - remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_right'), date_col > '2024-01-01' -); - -SELECT 'issue_1208_self_in_subquery'; -SELECT count() -FROM -( - SELECT string_col - FROM test_hybrid_issue_1208_1209_1422 - WHERE string_col IN - ( - SELECT DISTINCT string_col - FROM test_hybrid_issue_1208_1209_1422 - WHERE long_col > 1500 - ) -); - -SELECT 'issue_1209_join_mode_local'; -SELECT uniqExact(coalesce(h_string_col, m_string_col)) -FROM -( - SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long - FROM test_hybrid_issue_1208_1209_1422 AS h - FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col - SETTINGS object_storage_cluster_join_mode = 'local' -); - -SELECT 'issue_1209_join_mode_allow'; -SELECT uniqExact(coalesce(h_string_col, m_string_col)) -FROM -( - SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long - FROM test_hybrid_issue_1208_1209_1422 AS h - FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col - SETTINGS object_storage_cluster_join_mode = 'allow' -); - -SELECT 'issue_1422_hybrid_in_merge_tree_subquery'; -SELECT count() -FROM -( - SELECT string_col - FROM test_hybrid_issue_1208_1209_1422 - WHERE string_col IN - ( - SELECT DISTINCT string_col - FROM test_hybrid_issue_1208_1209_1422_mt - WHERE long_col > 1500 - ) -); - -DROP TABLE test_hybrid_issue_1208_1209_1422; -DROP TABLE test_hybrid_issue_1208_1209_1422_left; -DROP TABLE test_hybrid_issue_1208_1209_1422_right; -DROP TABLE test_hybrid_issue_1208_1209_1422_mt; diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference index 402cc360bae5..b58a303171aa 100644 --- a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference @@ -13,3 +13,12 @@ dist_prefer0_plan dist_prefer1_plan 11 12 21 22 +reordered_local +12 11 23 +22 21 43 +reordered_dist +12 11 23 +22 21 43 +reordered_dist_plan +12 11 23 +22 21 43 diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql index 848f35b0be14..32494b67da55 100644 --- a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql @@ -1,34 +1,44 @@ --- Plain Distributed (no Hybrid). Two nested ALIAS columns: a2 contains a1's subexpression, --- so planner CSE may reorder the remote header. Correct result must equal the single-node --- ('local') result across every transport variant. -DROP TABLE IF EXISTS t_local_03930; -DROP TABLE IF EXISTS t_dist_03930; +-- Nested `ALIAS` columns: `a2` contains `a1`'s subexpression, so planner CSE may reorder +-- the remote header. Check both alias-only and reordered/computed projections. +DROP TABLE IF EXISTS t_local_05058; +DROP TABLE IF EXISTS t_dist_05058; -CREATE TABLE t_local_03930 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +CREATE TABLE t_local_05058 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) ENGINE = MergeTree ORDER BY x; -INSERT INTO t_local_03930 VALUES (10), (20); +INSERT INTO t_local_05058 VALUES (10), (20); -CREATE TABLE t_dist_03930 AS t_local_03930 -ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03930); +CREATE TABLE t_dist_05058 AS t_local_05058 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05058); SELECT 'local'; -SELECT a1, a2 FROM t_local_03930 ORDER BY a1; +SELECT a1, a2 FROM t_local_05058 ORDER BY a1; SELECT 'dist_prefer0'; -SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SELECT a1, a2 FROM t_dist_05058 ORDER BY a1 SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; SELECT 'dist_prefer1'; -SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SELECT a1, a2 FROM t_dist_05058 ORDER BY a1 SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1; SELECT 'dist_prefer0_plan'; -SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SELECT a1, a2 FROM t_dist_05058 ORDER BY a1 SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; SELECT 'dist_prefer1_plan'; -SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SELECT a1, a2 FROM t_dist_05058 ORDER BY a1 SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; -DROP TABLE t_dist_03930; -DROP TABLE t_local_03930; +SELECT 'reordered_local'; +SELECT a2, a1, a1 + a2 AS s FROM t_local_05058 ORDER BY x; + +SELECT 'reordered_dist'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_05058 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0; + +SELECT 'reordered_dist_plan'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_05058 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_05058; +DROP TABLE t_local_05058; diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference deleted file mode 100644 index 97dd73615024..000000000000 --- a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference +++ /dev/null @@ -1,9 +0,0 @@ -local -1 10 100 -2 20 200 -merge_prefer0 -1 10 100 -2 20 200 -merge_prefer1 -1 10 100 -2 20 200 diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql deleted file mode 100644 index c42ad63adab4..000000000000 --- a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql +++ /dev/null @@ -1,64 +0,0 @@ --- Regression for the StorageMerge alias-output-naming fix. --- The bug: `Nested::splitName(name, reverse=true)` (used before this fix to strip the --- analyzer's `__tableN.` prefix from header column names) splits on the LAST dot, so for --- an analyzer identifier like `__table1.\`n.a\`` (a dotted column name wrapped in backticks --- by the analyzer) it returns the suffix `a\`` instead of `n.a`, leaving the --- `logical_name_to_header_name` map with broken keys. The lookup for `alias.name == "n.a"` --- then misses, the alias output falls back to the bare name `n.a`, and the downstream --- header-reconciliation step fills the expected `__table1.\`n.a\`` column with type --- defaults (zeros). Silent wrong data. --- --- Repro shape: Merge declares dotted column names explicitly (typical when matching a --- schema with Nested-style names), underlying storage has those columns as ALIAS, and --- the Distributed routing forces analyzer-prefixed names in the Merge level. Using a --- two-shard cluster with prefer_localhost_replica=0 reliably reproduces. - -DROP TABLE IF EXISTS test_04286_dotted_alias_local; -DROP TABLE IF EXISTS test_04286_dotted_alias_dist; -DROP TABLE IF EXISTS test_04286_dotted_alias_merge; - -CREATE TABLE test_04286_dotted_alias_local -( - id UInt32, - `n.a` UInt32 ALIAS id * 10, - `m.b` UInt32 ALIAS id * 100 -) -ENGINE = MergeTree -ORDER BY id; - -INSERT INTO test_04286_dotted_alias_local VALUES (1), (2); - -CREATE TABLE test_04286_dotted_alias_dist AS test_04286_dotted_alias_local -ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_04286_dotted_alias_local); - -CREATE TABLE test_04286_dotted_alias_merge -( - id UInt32, - `n.a` UInt32, - `m.b` UInt32 -) -ENGINE = Merge(currentDatabase(), '^test_04286_dotted_alias_dist$'); - -SELECT 'local'; -SELECT id, `n.a`, `m.b` -FROM test_04286_dotted_alias_local -GROUP BY id, `n.a`, `m.b` -ORDER BY id; - -SELECT 'merge_prefer0'; -SELECT id, `n.a`, `m.b` -FROM test_04286_dotted_alias_merge -GROUP BY id, `n.a`, `m.b` -ORDER BY id -SETTINGS prefer_localhost_replica = 0; - -SELECT 'merge_prefer1'; -SELECT id, `n.a`, `m.b` -FROM test_04286_dotted_alias_merge -GROUP BY id, `n.a`, `m.b` -ORDER BY id -SETTINGS prefer_localhost_replica = 1; - -DROP TABLE test_04286_dotted_alias_merge; -DROP TABLE test_04286_dotted_alias_dist; -DROP TABLE test_04286_dotted_alias_local; diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference deleted file mode 100644 index 286fa88dd6a4..000000000000 --- a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference +++ /dev/null @@ -1,35 +0,0 @@ -merge_tree_row_count -5 -iceberg_row_count -5 -hybrid_row_count -5 -true 8751 7291.267979503492 Frank 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 5313 8428.52 456.78 -false 3611 4492.090462838536 Isaac 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 4552 1554.795 456.78 -true 1519 3799.273006373374 Louis 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 8785 1248.2616 456.78 -true 1654 3801.2622503916614 Oliver 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3432 6701.752 456.78 -true 9044 2931.782814070929 William 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3733 7730.6836 456.78 -issue_1208_join_hybrid_mt_local -Frank 8751 8751 -Isaac 3611 3611 -Louis 1519 1519 -Oliver 1654 1654 -William 9044 9044 -issue_1208_join_hybrid_mt_allow -Frank 8751 8751 -Isaac 3611 3611 -Louis 1519 1519 -Oliver 1654 1654 -William 9044 9044 -issue_1208_join_hybrid_mt_iceberg_local -Frank 8751 8751 8751 -Isaac 3611 3611 3611 -Louis 1519 1519 1519 -Oliver 1654 1654 1654 -William 9044 9044 9044 -issue_1208_join_hybrid_mt_iceberg_allow -Frank 8751 8751 8751 -Isaac 3611 3611 3611 -Louis 1519 1519 1519 -Oliver 1654 1654 1654 -William 9044 9044 9044 diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql deleted file mode 100644 index d899f084ba4c..000000000000 --- a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql +++ /dev/null @@ -1,338 +0,0 @@ --- Tags: no-fasttest --- Hybrid table joined against a MergeTree and an Iceberg table, covering Altinity#1208, #1209 and --- #1422. Both `object_storage_cluster_join_mode` values must return the same rows: `'local'` used --- to raise UNKNOWN_IDENTIFIER here, and no longer does, so it is held to the `'allow'` result. --- --- Needs the stateless S3 mock on localhost:11111, and a server listening on 127.0.0.3 as well as --- 127.0.0.1 and 127.0.0.2, because `icebergCluster` below reads through --- `test_cluster_one_shard_three_replicas_localhost`. Without the third address the query still --- returns the right rows by failing over, but the connection warnings on stderr fail the test. -SET allow_experimental_hybrid_table = 1, - enable_analyzer = 1, - prefer_localhost_replica = 0, - iceberg_delete_data_on_drop = 1; - -DROP TABLE IF EXISTS hybrid_table; -DROP TABLE IF EXISTS merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; -DROP TABLE IF EXISTS iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; -DROP TABLE IF EXISTS merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; -DROP TABLE IF EXISTS hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; -DROP TABLE IF EXISTS merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; -DROP TABLE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; -DROP DATABASE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172; - -CREATE TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 -( - boolean_col Nullable(Bool), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Decimal(10, 2)) -) -ENGINE = MergeTree -PARTITION BY string_col -ORDER BY tuple() -SETTINGS index_granularity = 8192; - -INSERT INTO merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 VALUES - (true, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), - (true, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), - (true, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), - (true, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), - (false, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); - -SELECT 'merge_tree_row_count'; -SELECT count() FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; - -CREATE TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 -( - boolean_col Nullable(Int32), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Float64) -) -ENGINE = IcebergS3( - s3_conn, - filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table') -); - -INSERT INTO iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 VALUES - (1, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), - (1, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), - (1, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), - (1, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), - (0, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); - -SELECT 'iceberg_row_count'; -SELECT count() FROM iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; - -CREATE TABLE hybrid_table -( - boolean_col Nullable(Bool), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Decimal(10, 2)) -) -ENGINE = Hybrid( - remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), - date_col <= '2024-01-01', - icebergCluster( - 'test_cluster_one_shard_three_replicas_localhost', - concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), - 'test', - 'testtest' - ), - date_col > '2024-01-01' -); - -SELECT 'hybrid_row_count'; -SELECT count() FROM hybrid_table; - -CREATE TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 -( - boolean_col Nullable(Bool), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Decimal(10, 2)) -) -ENGINE = MergeTree -PARTITION BY string_col -ORDER BY tuple() -SETTINGS index_granularity = 8192; - -INSERT INTO merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 -SELECT * FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; - -CREATE DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; - -CREATE TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` -( - boolean_col Nullable(Int32), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Float64) -) -ENGINE = IcebergS3( - s3_conn, - filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table_39afe20a_d5d6_11f0_8208_e0c26496f172') -); - -INSERT INTO database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` -SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 -SELECT - toInt32(boolean_col), - long_col, - double_col, - string_col, - timestamp_col, - date_col, - time_col, - timestamptz_col, - integer_col, - float_col, - toFloat64(decimal_col) -FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; - -SELECT * -FROM hybrid_table -WHERE string_col IN -( - SELECT DISTINCT string_col - FROM hybrid_table - WHERE long_col > 1500 -) -ORDER BY string_col; - -SELECT 'issue_1208_join_hybrid_mt_local'; -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -ORDER BY h.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'local'; - -SELECT 'issue_1208_join_hybrid_mt_allow'; -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -ORDER BY h.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'allow'; - -SELECT 'issue_1208_join_hybrid_mt_iceberg_local'; -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long, - i.long_col AS iceberg_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col -ORDER BY h.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'local'; - -SELECT 'issue_1208_join_hybrid_mt_iceberg_allow'; -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long, - i.long_col AS iceberg_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col -ORDER BY h.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'allow'; - --- Exact issue-shape queries (no ORDER BY), deterministic output via FORMAT Null. -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'local' -FORMAT Null; - -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'allow' -FORMAT Null; - -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long, - i.long_col AS iceberg_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'local' -FORMAT Null; - -SELECT - h.string_col, - h.long_col AS hybrid_long, - m.long_col AS mt_long, - i.long_col AS iceberg_long -FROM hybrid_table AS h -FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col -FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col -LIMIT 10 -SETTINGS object_storage_cluster_join_mode = 'allow' -FORMAT Null; - -CREATE TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 -( - boolean_col Nullable(Bool), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Decimal(10, 2)) -) -ENGINE = MergeTree -PARTITION BY string_col -ORDER BY tuple() -SETTINGS index_granularity = 8192; - -INSERT INTO merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 -SELECT * FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; - -CREATE TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 -( - boolean_col Nullable(Bool), - long_col Nullable(Int64), - double_col Nullable(Float64), - string_col String, - timestamp_col Nullable(DateTime64(6)), - date_col Nullable(Date), - time_col Nullable(Int64), - timestamptz_col Nullable(DateTime64(6, 'UTC')), - integer_col Nullable(Int32), - float_col Nullable(Float32), - decimal_col Nullable(Decimal(10, 2)) -) -ENGINE = Hybrid( - remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), - date_col <= '2024-01-01', - icebergCluster( - 'test_cluster_one_shard_three_replicas_localhost', - concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), - 'test', - 'testtest' - ), - date_col > '2024-01-01' -); - -SELECT * -FROM hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 -WHERE string_col IN -( - SELECT DISTINCT string_col - FROM merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 - WHERE long_col > 1500 -) -FORMAT Null; - -DROP TABLE hybrid_table; -DROP TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; -DROP TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; -DROP TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; -DROP TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; -DROP TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; -DROP TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; -DROP DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference index 1e2e9b11750a..f48ffcc87bad 100644 --- a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference @@ -54,3 +54,17 @@ x a_str inner_c 10 aaaa 11 10 aaaa 11 10 aaaa 11 +prefer_localhost_replica_0_string_marker_on +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql index 84ad3cf170d0..44e2d818e936 100644 --- a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql @@ -1,3 +1,4 @@ +-- Compare the same `String` alias with the marker on and off; the setting must not change results. DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_outer; DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_inner; DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_local; @@ -91,6 +92,14 @@ SETTINGS parallel_replicas_local_plan = 0 FORMAT TSVWithNames; +SELECT 'prefer_localhost_replica_0_string_marker_on'; +SELECT x, a_str, inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, + enable_parallel_replicas = 0, max_parallel_replicas = 1, parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + DROP TABLE test_dod_alias_swap_no_marker_outer; DROP TABLE test_dod_alias_swap_no_marker_inner; DROP TABLE test_dod_alias_swap_no_marker_local; diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference index 228ac5f667f7..ee571cf0d1d4 100644 --- a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference @@ -18,3 +18,6 @@ x a_str inner_c 1 aaaa 2 2 aaaa 3 10 aaaa 11 +parallel_replicas_used +05062_pr_string 1 1 +05062_pr_uint64 1 1 diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql index 070330e98826..838da3e7c5fb 100644 --- a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql @@ -35,7 +35,6 @@ SETTINGS enable_analyzer = 1, enable_alias_marker = 0, enable_parallel_replicas = 0, - allow_experimental_parallel_reading_from_replicas = 0, max_parallel_replicas = 1, parallel_replicas_local_plan = 0, parallel_replicas_for_non_replicated_merge_tree = 1, @@ -50,7 +49,6 @@ SETTINGS enable_analyzer = 1, enable_alias_marker = 0, enable_parallel_replicas = 0, - allow_experimental_parallel_reading_from_replicas = 0, max_parallel_replicas = 1, parallel_replicas_local_plan = 0, parallel_replicas_for_non_replicated_merge_tree = 1, @@ -65,12 +63,13 @@ SETTINGS enable_analyzer = 1, enable_alias_marker = 0, enable_parallel_replicas = 2, - allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_local_plan = 1, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + log_queries = 1, + log_comment = '05062_pr_uint64' FORMAT TSVWithNames; SELECT 'pr_string'; @@ -81,14 +80,32 @@ SETTINGS enable_analyzer = 1, enable_alias_marker = 0, enable_parallel_replicas = 2, - allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_local_plan = 1, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, - cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + log_queries = 1, + log_comment = '05062_pr_string' FORMAT TSVWithNames; +SYSTEM FLUSH LOGS query_log; +-- Check the remote legs as well as the initiator: a correct result alone does not prove parallel reading. +SELECT 'parallel_replicas_used'; +SELECT log_comment, + countIf(ProfileEvents['ParallelReplicasQueryCount'] > 0) > 0, + countIf(ProfileEvents['ParallelReplicasUsedCount'] > 0) > 0 +FROM system.query_log +WHERE type = 'QueryFinish' AND is_initial_query = 0 + AND initial_query_id IN + ( + SELECT query_id FROM system.query_log + WHERE type = 'QueryFinish' AND is_initial_query + AND current_database = currentDatabase() + AND log_comment IN ('05062_pr_uint64', '05062_pr_string') + ) +GROUP BY log_comment ORDER BY log_comment; + DROP TABLE test_pr_dod_alias_swap_outer; DROP TABLE test_pr_dod_alias_swap_inner; DROP TABLE test_pr_dod_alias_swap_local; diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference index 97dd73615024..f69812ea180c 100644 --- a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference @@ -7,3 +7,12 @@ merge_prefer0 merge_prefer1 1 10 100 2 20 200 +multiple dotted local +1 1000 10000 +2 2000 20000 +multiple dotted merge_prefer0 +1 1000 10000 +2 2000 20000 +multiple dotted merge_prefer1 +1 1000 10000 +2 2000 20000 diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql index 79a985ba163d..d92def21e373 100644 --- a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql @@ -1,6 +1,5 @@ --- Values guard for a `Merge` over `Distributed` whose schema declares both a plain column and a --- dotted column whose tail is that plain name (`b` and `` `a.b` ``), both read as `ALIAS` columns. --- 05059 covers dotted names on their own; this shape is covered nowhere else. +-- A `Merge` over `Distributed` must preserve multiple dotted `ALIAS` names, including +-- a dotted name whose tail collides with a plain `ALIAS` name (`b` and `` `a.b` ``). -- -- `test_cluster_two_shards` reads the local table twice, so every query groups to dedup. @@ -12,7 +11,9 @@ CREATE TABLE t_local_05065 ( id UInt32, b UInt32 ALIAS id * 10, - `a.b` UInt32 ALIAS id * 100 + `a.b` UInt32 ALIAS id * 100, + `n.a` UInt32 ALIAS id * 1000, + `m.b` UInt32 ALIAS id * 10000 ) ENGINE = MergeTree ORDER BY id; @@ -26,7 +27,9 @@ CREATE TABLE t_merge_05065 ( id UInt32, b UInt32, - `a.b` UInt32 + `a.b` UInt32, + `n.a` UInt32, + `m.b` UInt32 ) ENGINE = Merge(currentDatabase(), '^t_dist_05065$'); @@ -50,6 +53,20 @@ GROUP BY id, b, `a.b` ORDER BY id SETTINGS prefer_localhost_replica = 1; +SELECT 'multiple dotted local'; +SELECT id, `n.a`, `m.b` FROM t_local_05065 +GROUP BY id, `n.a`, `m.b` ORDER BY id; + +SELECT 'multiple dotted merge_prefer0'; +SELECT id, `n.a`, `m.b` FROM t_merge_05065 +GROUP BY id, `n.a`, `m.b` ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +SELECT 'multiple dotted merge_prefer1'; +SELECT id, `n.a`, `m.b` FROM t_merge_05065 +GROUP BY id, `n.a`, `m.b` ORDER BY id +SETTINGS prefer_localhost_replica = 1; + DROP TABLE t_merge_05065; DROP TABLE t_dist_05065; DROP TABLE t_local_05065; From ede8dadd9647e40b5ea9a0bfcda3205471dbf918 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 29 Sep 2026 10:19:31 +0200 Subject: [PATCH 16/18] Keep pending-marker local test off parallel replicas The local `__aliasMarker` check must not ship to a replica when randomized settings enable parallel reads. Disable `enable_parallel_replicas` for that query only; leave the distributed boundary checks unchanged. --- .../0_stateless/05070_alias_marker_pending_boundaries.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql index b10810e31f7b..229b4c9d4cfc 100644 --- a/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql +++ b/tests/queries/0_stateless/05070_alias_marker_pending_boundaries.sql @@ -10,7 +10,8 @@ CREATE TABLE t_dist_05070 AS t_local_05070 ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05070); SELECT 'local_pending'; -SELECT __aliasMarker(1, 1, '__aliasMarker_pending_v1') FROM t_local_05070 ORDER BY x; +SELECT __aliasMarker(1, 1, '__aliasMarker_pending_v1') FROM t_local_05070 ORDER BY x +SETTINGS enable_parallel_replicas = 0; SELECT 'bad_token'; SELECT __aliasMarker(1, 1, 'wrong_token') FROM t_local_05070; -- { serverError BAD_ARGUMENTS } From 9a0a8fb671ffd5b0b690ebfff3a7ac71b20c0b24 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 29 Sep 2026 16:49:50 +0200 Subject: [PATCH 17/18] Pin execution modes for parallel-replica alias test The distributed-plan profile defaults to `serialize_query_plan = 1`, which bypasses parallel replica reads in the two measured `05062` queries. Randomized `automatic_parallel_replicas_mode = 2` suppresses those reads as well. Pin both settings so the participation assertions measure actual replica reads. CI triage: https://github.com/Altinity/ClickHouse/pull/2409#issuecomment-5890881999 --- .../05062_parallel_replicas_dod_alias_column_swap.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql index 838da3e7c5fb..c56ca47b41e3 100644 --- a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql @@ -55,6 +55,8 @@ SETTINGS cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' FORMAT TSVWithNames; +-- Serialized `Distributed` plans bypass parallel replicas; automatic mode 2 collects statistics only. +-- Pin both for queries whose replica participation is checked below. SELECT 'pr_uint64'; SELECT x, a_num, inner_c FROM test_pr_dod_alias_swap_outer @@ -67,7 +69,9 @@ SETTINGS parallel_replicas_local_plan = 1, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, + automatic_parallel_replicas_mode = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 0, log_queries = 1, log_comment = '05062_pr_uint64' FORMAT TSVWithNames; @@ -84,7 +88,9 @@ SETTINGS parallel_replicas_local_plan = 1, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_min_number_of_rows_per_replica = 0, + automatic_parallel_replicas_mode = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 0, log_queries = 1, log_comment = '05062_pr_string' FORMAT TSVWithNames; From 9a2a424bb1fe7f2d72910b978cdc83d85c91f47a Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Wed, 30 Sep 2026 10:07:46 +0200 Subject: [PATCH 18/18] Preserve `LowCardinality` payload types in `__aliasMarker` Disable the default `LowCardinality` adapter for `__aliasMarker`. The planner replaces the marker with its payload without conversion, so its inferred result type must preserve the payload's type. Previously, a lambda capturing a `LowCardinality(Nullable(Int64))` column could infer `Nullable(Int64)` but return the wrapped column, raising a `LOGICAL_ERROR` in `ColumnFunction::reduce`. Add a regression test covering lambda parameters and captured columns, including `NULL` values, locally and through both `Distributed` transports. The captured-column case reproduces the CI exception against the existing binary before the fix. Related: https://github.com/Altinity/ClickHouse/pull/2409#issuecomment-5904947261 --- src/Functions/identity.h | 2 + ...as_marker_low_cardinality_lambda.reference | 28 +++++++++++++ ...74_alias_marker_low_cardinality_lambda.sql | 42 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.reference create mode 100644 tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.sql diff --git a/src/Functions/identity.h b/src/Functions/identity.h index a5483624377a..54f63889ffa2 100644 --- a/src/Functions/identity.h +++ b/src/Functions/identity.h @@ -153,6 +153,8 @@ class FunctionAliasMarker : public IFunction /// Validate the token even if an argument is `NULL` or has type `Nothing`. bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForNothing() const override { return false; } + /// Preserve the payload type: the planner replaces the marker with its child without conversion. + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { diff --git a/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.reference b/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.reference new file mode 100644 index 000000000000..a2f29fe401b1 --- /dev/null +++ b/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.reference @@ -0,0 +1,28 @@ +payload_type +LowCardinality(Nullable(Int64)) LowCardinality(Nullable(Int64)) +LowCardinality(Nullable(Int64)) LowCardinality(Nullable(Int64)) +LowCardinality(Nullable(Int64)) LowCardinality(Nullable(Int64)) +lambda_local +[1] +[2] +[NULL] +lambda_capture_local +[1] +[2] +[NULL] +lambda_distributed_ast +[1] +[2] +[NULL] +lambda_capture_distributed_ast +[1] +[2] +[NULL] +lambda_distributed_plan +[1] +[2] +[NULL] +lambda_capture_distributed_plan +[1] +[2] +[NULL] diff --git a/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.sql b/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.sql new file mode 100644 index 000000000000..97d9a885c54d --- /dev/null +++ b/tests/queries/0_stateless/05074_alias_marker_low_cardinality_lambda.sql @@ -0,0 +1,42 @@ +-- The planner removes `__aliasMarker` without converting its payload, so type inference must preserve `LowCardinality`. +SET enable_analyzer = 1; +SET enable_alias_marker = 1; +SET allow_suspicious_low_cardinality_types = 1; + +DROP TABLE IF EXISTS alias_marker_lc_dist; +DROP TABLE IF EXISTS alias_marker_lc_local; + +CREATE TABLE alias_marker_lc_local (x LowCardinality(Nullable(Int64))) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO alias_marker_lc_local VALUES (1), (2), (NULL); + +CREATE TABLE alias_marker_lc_dist AS alias_marker_lc_local +ENGINE = Distributed(test_shard_localhost, currentDatabase(), alias_marker_lc_local); + +SELECT 'payload_type'; +SELECT toTypeName(x), toTypeName(__aliasMarker(x, x)) FROM alias_marker_lc_local ORDER BY x; + +SELECT 'lambda_local'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) FROM alias_marker_lc_local ORDER BY x; + +-- The fuzzer's asterisk expands to the table column captured by the lambda, not its parameter. +SELECT 'lambda_capture_local'; +SELECT arrayMap(lx -> __aliasMarker(*, lx), [x]) FROM alias_marker_lc_local ORDER BY x; + +SELECT 'lambda_distributed_ast'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) FROM alias_marker_lc_dist ORDER BY x +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0; + +SELECT 'lambda_capture_distributed_ast'; +SELECT arrayMap(lx -> __aliasMarker(*, lx), [x]) FROM alias_marker_lc_dist ORDER BY x +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0; + +SELECT 'lambda_distributed_plan'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) FROM alias_marker_lc_dist ORDER BY x +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'lambda_capture_distributed_plan'; +SELECT arrayMap(lx -> __aliasMarker(*, lx), [x]) FROM alias_marker_lc_dist ORDER BY x +SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE alias_marker_lc_dist; +DROP TABLE alias_marker_lc_local;