Skip to content

Commit 62154dc

Browse files
committed
test: cover dictionary NDV cap propagation
1 parent d176c35 commit 62154dc

2 files changed

Lines changed: 66 additions & 1 deletion

File tree

‎datafusion/proto/src/logical_plan/file_formats.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,10 +340,11 @@ mod parquet {
340340
}
341341

342342
#[test]
343-
fn enable_rle_to_dictionary_round_trips_through_codec() {
343+
fn rle_to_dictionary_options_round_trip_through_codec() {
344344
use datafusion_common::config::TableParquetOptions;
345345
let mut options = TableParquetOptions::default();
346346
options.global.enable_rle_to_dictionary = true;
347+
options.global.rle_to_dictionary_max_distinct_values = Some(200);
347348
let original: Arc<dyn FileFormatFactory> = Arc::new(ParquetFormatFactory {
348349
options: Some(options),
349350
});
@@ -364,6 +365,10 @@ mod parquet {
364365
.expect("parquet options");
365366

366367
assert!(decoded_options.global.enable_rle_to_dictionary);
368+
assert_eq!(
369+
decoded_options.global.rle_to_dictionary_max_distinct_values,
370+
Some(200)
371+
);
367372
}
368373
}
369374
}

‎datafusion/sqllogictest/test_files/parquet_rle_to_dictionary.slt‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,66 @@ Dictionary(Int32, Utf8) Utf8View
165165
statement ok
166166
DROP TABLE thresholded_dict;
167167

168+
# The cap uses the sum of row-group NDVs as a conservative upper bound.
169+
# Each row group has two distinct values, but their summed NDV exceeds the cap.
170+
query I
171+
COPY (
172+
SELECT column1 AS category
173+
FROM (VALUES ('a'), ('b'), ('a'), ('b'))
174+
)
175+
TO 'test_files/scratch/parquet_rle_to_dictionary/threshold_multi_rg.parquet'
176+
STORED AS PARQUET
177+
OPTIONS (
178+
'format.dictionary_enabled' true,
179+
'format.max_row_group_size' 2
180+
);
181+
----
182+
4
183+
184+
statement ok
185+
CREATE EXTERNAL TABLE threshold_multi_rg
186+
STORED AS PARQUET
187+
LOCATION 'test_files/scratch/parquet_rle_to_dictionary/threshold_multi_rg.parquet';
188+
189+
query T
190+
SELECT DISTINCT arrow_typeof(category) FROM threshold_multi_rg;
191+
----
192+
Utf8View
193+
194+
statement ok
195+
DROP TABLE threshold_multi_rg;
196+
197+
# A column remains dictionary encoded only when every file containing it
198+
# proves that its NDV is within the cap.
199+
query I
200+
COPY (SELECT column1 AS category FROM (VALUES ('a'), ('b')))
201+
TO 'test_files/scratch/parquet_rle_to_dictionary/threshold_mixed/low.parquet'
202+
STORED AS PARQUET
203+
OPTIONS ('format.dictionary_enabled' true);
204+
----
205+
2
206+
207+
query I
208+
COPY (SELECT column1 AS category FROM (VALUES ('a'), ('b'), ('c')))
209+
TO 'test_files/scratch/parquet_rle_to_dictionary/threshold_mixed/high.parquet'
210+
STORED AS PARQUET
211+
OPTIONS ('format.dictionary_enabled' true);
212+
----
213+
3
214+
215+
statement ok
216+
CREATE EXTERNAL TABLE threshold_mixed
217+
STORED AS PARQUET
218+
LOCATION 'test_files/scratch/parquet_rle_to_dictionary/threshold_mixed/';
219+
220+
query TI
221+
SELECT arrow_typeof(category), COUNT(*) FROM threshold_mixed GROUP BY arrow_typeof(category);
222+
----
223+
Utf8View 5
224+
225+
statement ok
226+
DROP TABLE threshold_mixed;
227+
168228
statement ok
169229
reset datafusion.execution.parquet.write_row_group_number_distinct_values;
170230

0 commit comments

Comments
 (0)