@@ -34,7 +34,7 @@ use crate::row_group_filter::RowGroupAccessPlanFilter;
3434use crate :: {
3535 BloomFilterStatistics , Int96Coercer , ParquetAccessPlan , ParquetFileMetrics ,
3636 ParquetFileReaderFactory , ParquetRowSelection , ParquetVirtualColumn ,
37- apply_file_schema_type_coercions ,
37+ schema_coercion :: apply_file_schema_type_coercions_with_rle ,
3838} ;
3939use arrow:: array:: RecordBatch ;
4040use arrow:: datatypes:: DataType ;
@@ -294,6 +294,8 @@ pub(super) struct ParquetMorselizer {
294294 /// container-level pruning. Sourced from
295295 /// `datafusion.execution.parquet.max_in_list_size`.
296296 pub max_in_list_size : usize ,
297+ /// Whether to ask arrow-rs to read promoted dictionary columns directly.
298+ pub enable_rle_to_dictionary : bool ,
297299 /// Whether to read row groups in reverse order
298300 pub reverse_row_groups : bool ,
299301 /// Optional sort order used to reorder row groups by their min/max statistics.
@@ -457,6 +459,7 @@ struct PreparedParquetOpen {
457459 predicate_creation_errors : Count ,
458460 max_predicate_cache_size : Option < usize > ,
459461 max_in_list_size : usize ,
462+ enable_rle_to_dictionary : bool ,
460463 reverse_row_groups : bool ,
461464 sort_order_for_reorder : Option < LexOrdering > ,
462465 preserve_order : bool ,
@@ -854,6 +857,7 @@ impl ParquetMorselizer {
854857 predicate_creation_errors,
855858 max_predicate_cache_size : self . max_predicate_cache_size ,
856859 max_in_list_size : self . max_in_list_size ,
860+ enable_rle_to_dictionary : self . enable_rle_to_dictionary ,
857861 reverse_row_groups : self . reverse_row_groups ,
858862 sort_order_for_reorder : self . sort_order_for_reorder . clone ( ) ,
859863 preserve_order : self . preserve_order ,
@@ -962,9 +966,10 @@ impl MetadataLoadedParquetOpen {
962966 // desired schema (for example if we want to instruct the parquet
963967 // reader to read strings using Utf8View instead). Update if necessary
964968 let mut metadata_dirty = false ;
965- if let Some ( merged) = apply_file_schema_type_coercions (
969+ if let Some ( merged) = apply_file_schema_type_coercions_with_rle (
966970 & prepared. logical_file_schema ,
967971 & physical_file_schema,
972+ prepared. enable_rle_to_dictionary ,
968973 ) {
969974 physical_file_schema = Arc :: new ( merged) ;
970975 options = options. with_schema ( Arc :: clone ( & physical_file_schema) ) ;
@@ -1807,6 +1812,7 @@ mod test {
18071812 coerce_int96 : Option < TimeUnit > ,
18081813 max_predicate_cache_size : Option < usize > ,
18091814 max_in_list_size : usize ,
1815+ enable_rle_to_dictionary : bool ,
18101816 reverse_row_groups : bool ,
18111817 preserve_order : bool ,
18121818 }
@@ -2018,6 +2024,7 @@ mod test {
20182024 coerce_int96 : None ,
20192025 max_predicate_cache_size : None ,
20202026 max_in_list_size : MAX_IN_LIST_SIZE ,
2027+ enable_rle_to_dictionary : false ,
20212028 reverse_row_groups : false ,
20222029 preserve_order : false ,
20232030 }
@@ -2093,6 +2100,11 @@ mod test {
20932100 self
20942101 }
20952102
2103+ fn with_enable_rle_to_dictionary ( mut self , enable : bool ) -> Self {
2104+ self . enable_rle_to_dictionary = enable;
2105+ self
2106+ }
2107+
20962108 fn with_metrics ( mut self , metrics : ExecutionPlanMetricsSet ) -> Self {
20972109 self . metrics = metrics;
20982110 self
@@ -2196,6 +2208,7 @@ mod test {
21962208 encryption_factory : None ,
21972209 max_predicate_cache_size : self . max_predicate_cache_size ,
21982210 max_in_list_size : self . max_in_list_size ,
2211+ enable_rle_to_dictionary : self . enable_rle_to_dictionary ,
21992212 reverse_row_groups : self . reverse_row_groups ,
22002213 sort_order_for_reorder : None ,
22012214 virtual_state,
@@ -3920,4 +3933,59 @@ mod test {
39203933 assert_eq ! ( rows, 5 ) ;
39213934 }
39223935 }
3936+
3937+ async fn collect_batches (
3938+ morselizer : & ParquetMorselizer ,
3939+ file : PartitionedFile ,
3940+ ) -> Vec < RecordBatch > {
3941+ let mut stream = open_file ( morselizer, file) . await . unwrap ( ) ;
3942+ let mut batches = Vec :: new ( ) ;
3943+ while let Some ( batch) = stream. next ( ) . await {
3944+ batches. push ( batch. unwrap ( ) ) ;
3945+ }
3946+ batches
3947+ }
3948+
3949+ // Proves the opener passes a promoted binary Dictionary schema to arrow-rs.
3950+ #[ tokio:: test]
3951+ async fn test_rle_binary_column_promotion ( ) {
3952+ let store = Arc :: new ( InMemory :: new ( ) ) as Arc < dyn ObjectStore > ;
3953+ let bin_schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
3954+ "payload" ,
3955+ DataType :: Binary ,
3956+ true ,
3957+ ) ] ) ) ;
3958+ let values =
3959+ Arc :: new ( arrow:: array:: BinaryArray :: from_vec ( vec ! [ b"a" , b"b" , b"a" ] ) ) ;
3960+ let batch = RecordBatch :: try_new ( Arc :: clone ( & bin_schema) , vec ! [ values] ) . unwrap ( ) ;
3961+ let props = WriterProperties :: builder ( )
3962+ . set_dictionary_enabled ( true )
3963+ . build ( ) ;
3964+ let bin_size = write_parquet_batches (
3965+ Arc :: clone ( & store) ,
3966+ "binary.parquet" ,
3967+ vec ! [ batch] ,
3968+ Some ( props) ,
3969+ )
3970+ . await ;
3971+ let dict_bin_schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
3972+ "payload" ,
3973+ DataType :: Dictionary ( Box :: new( DataType :: Int32 ) , Box :: new( DataType :: Binary ) ) ,
3974+ true ,
3975+ ) ] ) ) ;
3976+ let morselizer = ParquetMorselizerBuilder :: new ( )
3977+ . with_store ( Arc :: clone ( & store) )
3978+ . with_schema ( Arc :: clone ( & dict_bin_schema) )
3979+ . with_enable_rle_to_dictionary ( true )
3980+ . build ( ) ;
3981+ let batches = collect_batches (
3982+ & morselizer,
3983+ PartitionedFile :: new ( "binary.parquet" . to_string ( ) , bin_size as u64 ) ,
3984+ )
3985+ . await ;
3986+ assert_eq ! (
3987+ batches[ 0 ] . schema( ) . field( 0 ) . data_type( ) ,
3988+ & DataType :: Dictionary ( Box :: new( DataType :: Int32 ) , Box :: new( DataType :: Binary ) )
3989+ ) ;
3990+ }
39233991}
0 commit comments