Skip to content

Commit f7fe23e

Browse files
committed
feat: introduce optional RLE-dictionary preservation for Parquet reads
When datafusion.execution.parquet.enable_rle_to_dictionary is true (the default), DataFusion inspects Parquet footer metadata during schema inference and promotes top-level string/binary columns that have dictionary pages to Arrow Dictionary(Int32, Utf8) / Dictionary(Int32, Binary) types. Mixed dictionary/plain directories are normalized before schema merge so the types are consistent across files. At scan time the promoted schema is passed to arrow-rs so those columns are read as dictionary arrays directly. Tables with a user-supplied schema are not promoted because DataFusion does not read footer metadata at DDL time. Closes apache#24112, part of apache#24111
1 parent 8332cfa commit f7fe23e

26 files changed

Lines changed: 1016 additions & 116 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1293,6 +1293,17 @@ config_namespace! {
12931293
/// Defaults to 20.
12941294
pub max_in_list_size: usize, default = 20
12951295

1296+
/// (reading) If true, top-level string and binary Parquet columns with
1297+
/// dictionary pages are inferred and scanned as
1298+
/// `Dictionary<Int32, Utf8>` / `Dictionary<Int32, Binary>` instead of
1299+
/// their plain value type.
1300+
///
1301+
/// This applies only when DataFusion infers the table schema. Tables with
1302+
/// a user-supplied schema are not promoted because the Parquet footer is
1303+
/// not read at DDL time, so dictionary pages cannot be detected per column.
1304+
/// See <https://github.com/apache/datafusion/issues/24112>
1305+
pub enable_rle_to_dictionary: bool, default = true
1306+
12961307
// The following options affect writing to parquet files
12971308
// and map to parquet::file::properties::WriterProperties
12981309

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ impl ParquetOptions {
249249
skip_arrow_metadata: _,
250250
max_predicate_cache_size: _,
251251
max_in_list_size: _,
252+
enable_rle_to_dictionary: _,
252253
} = self;
253254

254255
let mut builder = WriterProperties::builder()
@@ -509,6 +510,7 @@ mod tests {
509510
coerce_int96_tz: None,
510511
max_predicate_cache_size: defaults.max_predicate_cache_size,
511512
content_defined_chunking: defaults.content_defined_chunking.clone(),
513+
enable_rle_to_dictionary: defaults.enable_rle_to_dictionary,
512514
}
513515
}
514516

@@ -631,6 +633,8 @@ mod tests {
631633
coerce_int96: None,
632634
coerce_int96_tz: None,
633635
content_defined_chunking: props.content_defined_chunking().into(),
636+
enable_rle_to_dictionary: global_options_defaults
637+
.enable_rle_to_dictionary,
634638
},
635639
column_specific_options,
636640
key_value_metadata,

‎datafusion/core/src/datasource/file_format/parquet.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -544,8 +544,13 @@ mod tests {
544544
let (files, _file_names) = store_parquet(vec![batch1], false).await?;
545545

546546
let state = SessionContext::new().state();
547-
// Make metadata size hint None to keep original behavior
548-
let format = ParquetFormat::default().with_metadata_size_hint(None);
547+
// Make metadata size hint None and disable RLE-to-dictionary so this
548+
// test focuses on dictionary column statistics without dict-promotion.
549+
let mut parquet_opts = TableParquetOptions::default();
550+
parquet_opts.global.enable_rle_to_dictionary = false;
551+
let format = ParquetFormat::default()
552+
.with_options(parquet_opts)
553+
.with_metadata_size_hint(None);
549554
let _schema = format.infer_schema(&state, &store.upcast(), &files).await?;
550555
assert_eq!(store.request_count(), 3);
551556
// No increase, cache being used.
@@ -619,8 +624,11 @@ mod tests {
619624

620625
let mut state = SessionContext::new().state();
621626
state = set_view_state(state, force_views);
627+
let mut parquet_opts = TableParquetOptions::default();
628+
parquet_opts.global.schema_force_view_types = force_views;
629+
parquet_opts.global.enable_rle_to_dictionary = false;
622630
let format = ParquetFormat::default()
623-
.with_force_view_types(force_views)
631+
.with_options(parquet_opts)
624632
.with_metadata_size_hint(None);
625633
let schema = format.infer_schema(&state, &store.upcast(), &files).await?;
626634
assert_eq!(store.request_count(), 6);
@@ -791,6 +799,9 @@ mod tests {
791799
fn set_view_state(mut state: SessionState, use_views: bool) -> SessionState {
792800
let mut options = TableParquetOptions::default();
793801
options.global.schema_force_view_types = use_views;
802+
// Disable dictionary promotion so view-type tests are not affected by
803+
// the RLE-to-dictionary feature.
804+
options.global.enable_rle_to_dictionary = false;
794805
state
795806
.register_file_format(
796807
Arc::new(ParquetFormatFactory::new_with_options(options)),

‎datafusion/core/tests/parquet/schema_coercion.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -154,14 +154,10 @@ pub fn store_parquet(
154154
.into_iter()
155155
.map(|batch| {
156156
let mut output = NamedTempFile::new().expect("creating temp file");
157-
158-
let builder = WriterProperties::builder();
159-
let props = builder.build();
160-
157+
let props = WriterProperties::builder().build();
161158
let mut writer =
162159
ArrowWriter::try_new(&mut output, batch.schema(), Some(props))
163160
.expect("creating writer");
164-
165161
writer.write(&batch).expect("Writing batch");
166162
writer.close().unwrap();
167163
output

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,9 @@ impl FileFormat for ParquetFormat {
366366
.with_file_metadata_cache(Some(Arc::clone(&file_metadata_cache)))
367367
.with_coerce_int96(coerce_int96)
368368
.with_coerce_int96_tz(coerce_int96_tz.clone())
369+
.with_enable_rle_to_dictionary(
370+
self.options.global.enable_rle_to_dictionary,
371+
)
369372
.fetch_schema_with_location()
370373
.await?;
371374
Ok::<_, DataFusionError>(result)
@@ -402,7 +405,13 @@ impl FileFormat for ParquetFormat {
402405
}
403406
drop(seen);
404407

405-
let schemas = schemas.into_iter().map(|(_, schema)| schema);
408+
// Drop file locations before merging, and normalize dictionary-promoted
409+
// schemas so mixed dictionary/plain files merge consistently.
410+
let mut schemas: Vec<Schema> =
411+
schemas.into_iter().map(|(_, schema)| schema).collect();
412+
if self.options.global.enable_rle_to_dictionary {
413+
schemas = crate::schema_coercion::uniform_dict_schemas(schemas);
414+
}
406415

407416
let schema = if self.skip_metadata() {
408417
Schema::try_merge(clear_metadata(schemas))
@@ -524,7 +533,7 @@ impl FileFormat for ParquetFormat {
524533
source = source.with_parquet_file_reader_factory(cached_parquet_read_factory);
525534

526535
if let Some(metadata_size_hint) = metadata_size_hint {
527-
source = source.with_metadata_size_hint(metadata_size_hint)
536+
source = source.with_metadata_size_hint(metadata_size_hint);
528537
}
529538

530539
source = self.set_source_encryption_factory(source, state)?;
@@ -738,6 +747,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
738747
compression_opt: global_options.global.compression.map(|compression| {
739748
parquet_options::CompressionOpt::Compression(compression)
740749
}),
750+
enable_rle_to_dictionary: global_options.global.enable_rle_to_dictionary,
741751
dictionary_enabled_opt: global_options.global.dictionary_enabled.map(|enabled| {
742752
parquet_options::DictionaryEnabledOpt::DictionaryEnabled(enabled)
743753
}),

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ use crate::{Int96Coercer, apply_file_schema_type_coercions};
2323
use arrow::array::{Array, ArrayRef, BooleanArray};
2424
use arrow::compute::kernels::cmp::eq;
2525
use arrow::compute::{and, sum};
26-
use arrow::datatypes::{DataType, Schema, SchemaRef, TimeUnit};
26+
use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit};
2727
use datafusion_common::encryption::FileDecryptionProperties;
2828
use datafusion_common::stats::Precision;
2929
use datafusion_common::{
30-
ColumnStatistics, DataFusionError, HashMap, Result, ScalarValue, Statistics,
30+
ColumnStatistics, DataFusionError, HashMap, HashSet, Result, ScalarValue, Statistics,
3131
internal_datafusion_err,
3232
};
3333
use datafusion_execution::cache::cache_manager::{
@@ -91,6 +91,9 @@ pub struct DFParquetMetadata<'a> {
9191
pub coerce_int96: Option<TimeUnit>,
9292
/// Optional timezone applied to INT96-coerced timestamps.
9393
pub coerce_int96_tz: Option<Arc<str>>,
94+
/// Whether top-level string/binary columns with dictionary pages should be
95+
/// promoted to `Dictionary(Int32, ...)` in the inferred schema.
96+
enable_rle_to_dictionary: bool,
9497
}
9598

9699
impl<'a> DFParquetMetadata<'a> {
@@ -108,9 +111,17 @@ impl<'a> DFParquetMetadata<'a> {
108111
page_index_policy: None,
109112
coerce_int96: None,
110113
coerce_int96_tz: None,
114+
enable_rle_to_dictionary: false,
111115
}
112116
}
113117

118+
/// Promote top-level string/binary columns with dictionary pages to
119+
/// `Dictionary(Int32, ...)` in the schema returned by [`Self::fetch_schema`].
120+
pub fn with_enable_rle_to_dictionary(mut self, enable: bool) -> Self {
121+
self.enable_rle_to_dictionary = enable;
122+
self
123+
}
124+
114125
/// Set a hint for the number of trailing bytes to prefetch from the end
115126
/// of the file, equivalent to
116127
/// [`ParquetMetaDataReader::with_prefetch_hint`].
@@ -384,6 +395,74 @@ impl<'a> DFParquetMetadata<'a> {
384395
.coerce()
385396
})
386397
.unwrap_or(schema);
398+
399+
// Promote before force_view_types so dictionary columns survive the
400+
// Utf8 -> Utf8View pass.
401+
let schema = if self.enable_rle_to_dictionary {
402+
let rle_cols: HashSet<String> = {
403+
let schema_descr = file_metadata.schema_descr();
404+
metadata
405+
.row_groups()
406+
.iter()
407+
.flat_map(|rg| {
408+
rg.columns()
409+
.iter()
410+
.enumerate()
411+
.filter_map(|(col_idx, col)| {
412+
col.dictionary_page_offset()?;
413+
let desc = schema_descr.column(col_idx);
414+
// Only promote top-level columns. For nested columns
415+
// the Parquet leaf name (e.g. "element" inside a List)
416+
// does not match the Arrow top-level field name (e.g.
417+
// "tags"), so using the leaf name here would miss the
418+
// target field or promote an unrelated one.
419+
let parts = desc.path().parts();
420+
(parts.len() == 1).then(|| parts[0].clone())
421+
})
422+
})
423+
.collect()
424+
};
425+
if rle_cols.is_empty() {
426+
schema
427+
} else {
428+
let promoted: Vec<_> = schema
429+
.fields()
430+
.iter()
431+
.map(|field| {
432+
if !rle_cols.contains(field.name()) {
433+
return Arc::clone(field);
434+
}
435+
let dict_value_type = match field.data_type() {
436+
DataType::Utf8 => Some(DataType::Utf8),
437+
DataType::LargeUtf8 => Some(DataType::LargeUtf8),
438+
DataType::Binary => Some(DataType::Binary),
439+
DataType::LargeBinary => Some(DataType::LargeBinary),
440+
_ => None,
441+
};
442+
dict_value_type.map_or_else(
443+
|| Arc::clone(field),
444+
|value_type| {
445+
Arc::new(
446+
Field::new(
447+
field.name(),
448+
DataType::Dictionary(
449+
Box::new(DataType::Int32),
450+
Box::new(value_type),
451+
),
452+
field.is_nullable(),
453+
)
454+
.with_metadata(field.metadata().clone()),
455+
)
456+
},
457+
)
458+
})
459+
.collect();
460+
Schema::new_with_metadata(promoted, schema.metadata().clone())
461+
}
462+
} else {
463+
schema
464+
};
465+
387466
Ok(schema)
388467
}
389468

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 70 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use crate::row_group_filter::RowGroupAccessPlanFilter;
3434
use crate::{
3535
BloomFilterStatistics, Int96Coercer, ParquetAccessPlan, ParquetFileMetrics,
3636
ParquetFileReaderFactory, ParquetRowSelection, ParquetVirtualColumn,
37-
apply_file_schema_type_coercions,
37+
schema_coercion::apply_file_schema_type_coercions_with_rle,
3838
};
3939
use arrow::array::RecordBatch;
4040
use arrow::datatypes::DataType;
@@ -294,6 +294,8 @@ pub(super) struct ParquetMorselizer {
294294
/// container-level pruning. Sourced from
295295
/// `datafusion.execution.parquet.max_in_list_size`.
296296
pub max_in_list_size: usize,
297+
/// Whether to ask arrow-rs to read promoted dictionary columns directly.
298+
pub enable_rle_to_dictionary: bool,
297299
/// Whether to read row groups in reverse order
298300
pub reverse_row_groups: bool,
299301
/// Optional sort order used to reorder row groups by their min/max statistics.
@@ -457,6 +459,7 @@ struct PreparedParquetOpen {
457459
predicate_creation_errors: Count,
458460
max_predicate_cache_size: Option<usize>,
459461
max_in_list_size: usize,
462+
enable_rle_to_dictionary: bool,
460463
reverse_row_groups: bool,
461464
sort_order_for_reorder: Option<LexOrdering>,
462465
preserve_order: bool,
@@ -854,6 +857,7 @@ impl ParquetMorselizer {
854857
predicate_creation_errors,
855858
max_predicate_cache_size: self.max_predicate_cache_size,
856859
max_in_list_size: self.max_in_list_size,
860+
enable_rle_to_dictionary: self.enable_rle_to_dictionary,
857861
reverse_row_groups: self.reverse_row_groups,
858862
sort_order_for_reorder: self.sort_order_for_reorder.clone(),
859863
preserve_order: self.preserve_order,
@@ -962,9 +966,10 @@ impl MetadataLoadedParquetOpen {
962966
// desired schema (for example if we want to instruct the parquet
963967
// reader to read strings using Utf8View instead). Update if necessary
964968
let mut metadata_dirty = false;
965-
if let Some(merged) = apply_file_schema_type_coercions(
969+
if let Some(merged) = apply_file_schema_type_coercions_with_rle(
966970
&prepared.logical_file_schema,
967971
&physical_file_schema,
972+
prepared.enable_rle_to_dictionary,
968973
) {
969974
physical_file_schema = Arc::new(merged);
970975
options = options.with_schema(Arc::clone(&physical_file_schema));
@@ -1807,6 +1812,7 @@ mod test {
18071812
coerce_int96: Option<TimeUnit>,
18081813
max_predicate_cache_size: Option<usize>,
18091814
max_in_list_size: usize,
1815+
enable_rle_to_dictionary: bool,
18101816
reverse_row_groups: bool,
18111817
preserve_order: bool,
18121818
}
@@ -2018,6 +2024,7 @@ mod test {
20182024
coerce_int96: None,
20192025
max_predicate_cache_size: None,
20202026
max_in_list_size: MAX_IN_LIST_SIZE,
2027+
enable_rle_to_dictionary: false,
20212028
reverse_row_groups: false,
20222029
preserve_order: false,
20232030
}
@@ -2093,6 +2100,11 @@ mod test {
20932100
self
20942101
}
20952102

2103+
fn with_enable_rle_to_dictionary(mut self, enable: bool) -> Self {
2104+
self.enable_rle_to_dictionary = enable;
2105+
self
2106+
}
2107+
20962108
fn with_metrics(mut self, metrics: ExecutionPlanMetricsSet) -> Self {
20972109
self.metrics = metrics;
20982110
self
@@ -2196,6 +2208,7 @@ mod test {
21962208
encryption_factory: None,
21972209
max_predicate_cache_size: self.max_predicate_cache_size,
21982210
max_in_list_size: self.max_in_list_size,
2211+
enable_rle_to_dictionary: self.enable_rle_to_dictionary,
21992212
reverse_row_groups: self.reverse_row_groups,
22002213
sort_order_for_reorder: None,
22012214
virtual_state,
@@ -3920,4 +3933,59 @@ mod test {
39203933
assert_eq!(rows, 5);
39213934
}
39223935
}
3936+
3937+
async fn collect_batches(
3938+
morselizer: &ParquetMorselizer,
3939+
file: PartitionedFile,
3940+
) -> Vec<RecordBatch> {
3941+
let mut stream = open_file(morselizer, file).await.unwrap();
3942+
let mut batches = Vec::new();
3943+
while let Some(batch) = stream.next().await {
3944+
batches.push(batch.unwrap());
3945+
}
3946+
batches
3947+
}
3948+
3949+
// Proves the opener passes a promoted binary Dictionary schema to arrow-rs.
3950+
#[tokio::test]
3951+
async fn test_rle_binary_column_promotion() {
3952+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
3953+
let bin_schema = Arc::new(Schema::new(vec![Field::new(
3954+
"payload",
3955+
DataType::Binary,
3956+
true,
3957+
)]));
3958+
let values =
3959+
Arc::new(arrow::array::BinaryArray::from_vec(vec![b"a", b"b", b"a"]));
3960+
let batch = RecordBatch::try_new(Arc::clone(&bin_schema), vec![values]).unwrap();
3961+
let props = WriterProperties::builder()
3962+
.set_dictionary_enabled(true)
3963+
.build();
3964+
let bin_size = write_parquet_batches(
3965+
Arc::clone(&store),
3966+
"binary.parquet",
3967+
vec![batch],
3968+
Some(props),
3969+
)
3970+
.await;
3971+
let dict_bin_schema = Arc::new(Schema::new(vec![Field::new(
3972+
"payload",
3973+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Binary)),
3974+
true,
3975+
)]));
3976+
let morselizer = ParquetMorselizerBuilder::new()
3977+
.with_store(Arc::clone(&store))
3978+
.with_schema(Arc::clone(&dict_bin_schema))
3979+
.with_enable_rle_to_dictionary(true)
3980+
.build();
3981+
let batches = collect_batches(
3982+
&morselizer,
3983+
PartitionedFile::new("binary.parquet".to_string(), bin_size as u64),
3984+
)
3985+
.await;
3986+
assert_eq!(
3987+
batches[0].schema().field(0).data_type(),
3988+
&DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Binary))
3989+
);
3990+
}
39233991
}

0 commit comments

Comments
 (0)