Skip to content

Commit e61aeb2

Browse files
committed
address review
1 parent 3ed9076 commit e61aeb2

1 file changed

Lines changed: 35 additions & 0 deletions

File tree

‎datafusion/sqllogictest/test_files/parquet_ndv_write.slt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,41 @@ physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/
8383
statement ok
8484
DROP TABLE with_ndv;
8585

86+
######
87+
# Dictionary column NDV is unreliable (arrow-rs hashes dictionary keys, not
88+
# logical values), so stats-based `COUNT(DISTINCT)` folding must not use it.
89+
# Pin this: the written NDV for `d` is wrong (2), but the query still returns
90+
# the correct 6 because the Inexact precision prevents the stats fold.
91+
######
92+
93+
statement ok
94+
set datafusion.execution.batch_size = 2;
95+
96+
query I
97+
COPY (SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS d, column2 AS x
98+
FROM (VALUES ('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6)))
99+
TO 'test_files/scratch/parquet_ndv_write/dict.parquet'
100+
STORED AS PARQUET;
101+
----
102+
6
103+
104+
statement ok
105+
CREATE EXTERNAL TABLE dict_ndv
106+
STORED AS PARQUET
107+
LOCATION 'test_files/scratch/parquet_ndv_write/dict.parquet';
108+
109+
# NDV written for `d` is wrong (2), so it must not be used as Exact
110+
query II
111+
SELECT count(DISTINCT d), count(DISTINCT x) FROM dict_ndv;
112+
----
113+
6 6
114+
115+
statement ok
116+
DROP TABLE dict_ndv;
117+
118+
statement ok
119+
RESET datafusion.execution.batch_size;
120+
86121
# Reset settings
87122
statement ok
88123
RESET datafusion.execution.parquet.write_row_group_number_distinct_values;

0 commit comments

Comments
 (0)