@@ -680,3 +680,129 @@ pub fn statistics_from_parquet_meta_calc(
680680) -> Result < Statistics > {
681681 DFParquetMetadata :: statistics_from_parquet_metadata ( metadata, & table_schema)
682682}
683+
684+ #[ cfg( feature = "proto" ) ]
685+ use datafusion_proto_models:: protobuf:: { self , parquet_column_options, parquet_options} ;
686+
687+ /// Encode a [`ParquetFormatFactory`]'s options as their protobuf form.
688+ ///
689+ /// The reverse direction is `TryFrom<&protobuf::TableParquetOptions> for
690+ /// TableParquetOptions` in `datafusion-proto-models`: `TableParquetOptions` is
691+ /// a `datafusion-common` type, so that half cannot live here.
692+ #[ cfg( feature = "proto" ) ]
693+ impl From < & ParquetFormatFactory > for protobuf:: TableParquetOptions {
694+ fn from ( factory : & ParquetFormatFactory ) -> Self {
695+ let global_options = if let Some ( ref options) = factory. options {
696+ options. clone ( )
697+ } else {
698+ return protobuf:: TableParquetOptions :: default ( ) ;
699+ } ;
700+
701+ let column_specific_options = global_options. column_specific_options ;
702+ protobuf:: TableParquetOptions {
703+ global : Some ( protobuf:: ParquetOptions {
704+ enable_page_index : global_options. global . enable_page_index ,
705+ pruning : global_options. global . pruning ,
706+ skip_metadata : global_options. global . skip_metadata ,
707+ metadata_size_hint_opt : global_options. global . metadata_size_hint . map ( |size| {
708+ parquet_options:: MetadataSizeHintOpt :: MetadataSizeHint ( size as u64 )
709+ } ) ,
710+ pushdown_filters : global_options. global . pushdown_filters ,
711+ reorder_filters : global_options. global . reorder_filters ,
712+ force_filter_selections : global_options. global . force_filter_selections ,
713+ data_pagesize_limit : global_options. global . data_pagesize_limit as u64 ,
714+ write_batch_size : global_options. global . write_batch_size as u64 ,
715+ writer_version : global_options. global . writer_version . to_string ( ) ,
716+ compression_opt : global_options. global . compression . map ( |compression| {
717+ parquet_options:: CompressionOpt :: Compression ( compression)
718+ } ) ,
719+ dictionary_enabled_opt : global_options. global . dictionary_enabled . map ( |enabled| {
720+ parquet_options:: DictionaryEnabledOpt :: DictionaryEnabled ( enabled)
721+ } ) ,
722+ dictionary_page_size_limit : global_options. global . dictionary_page_size_limit as u64 ,
723+ statistics_enabled_opt : global_options. global . statistics_enabled . map ( |enabled| {
724+ parquet_options:: StatisticsEnabledOpt :: StatisticsEnabled ( enabled)
725+ } ) ,
726+ max_row_group_size : global_options. global . max_row_group_size as u64 ,
727+ max_in_list_size : global_options. global . max_in_list_size as u64 ,
728+ created_by : global_options. global . created_by . clone ( ) ,
729+ column_index_truncate_length_opt : global_options. global . column_index_truncate_length . map ( |length| {
730+ parquet_options:: ColumnIndexTruncateLengthOpt :: ColumnIndexTruncateLength ( length as u64 )
731+ } ) ,
732+ statistics_truncate_length_opt : global_options. global . statistics_truncate_length . map ( |length| {
733+ parquet_options:: StatisticsTruncateLengthOpt :: StatisticsTruncateLength ( length as u64 )
734+ } ) ,
735+ data_page_row_count_limit : global_options. global . data_page_row_count_limit as u64 ,
736+ encoding_opt : global_options. global . encoding . map ( |encoding| {
737+ parquet_options:: EncodingOpt :: Encoding ( encoding)
738+ } ) ,
739+ bloom_filter_on_read : global_options. global . bloom_filter_on_read ,
740+ bloom_filter_on_write : global_options. global . bloom_filter_on_write ,
741+ bloom_filter_fpp_opt : global_options. global . bloom_filter_fpp . map ( |fpp| {
742+ parquet_options:: BloomFilterFppOpt :: BloomFilterFpp ( fpp)
743+ } ) ,
744+ bloom_filter_ndv_opt : global_options. global . bloom_filter_ndv . map ( |ndv| {
745+ parquet_options:: BloomFilterNdvOpt :: BloomFilterNdv ( ndv)
746+ } ) ,
747+ allow_single_file_parallelism : global_options. global . allow_single_file_parallelism ,
748+ maximum_parallel_row_group_writers : global_options. global . maximum_parallel_row_group_writers as u64 ,
749+ maximum_buffered_record_batches_per_stream : global_options. global . maximum_buffered_record_batches_per_stream as u64 ,
750+ schema_force_view_types : global_options. global . schema_force_view_types ,
751+ binary_as_string : global_options. global . binary_as_string ,
752+ skip_arrow_metadata : global_options. global . skip_arrow_metadata ,
753+ coerce_int96_opt : global_options. global . coerce_int96 . map ( |compression| {
754+ parquet_options:: CoerceInt96Opt :: CoerceInt96 ( compression)
755+ } ) ,
756+ coerce_int96_tz_opt : global_options. global . coerce_int96_tz . map ( |tz| {
757+ parquet_options:: CoerceInt96TzOpt :: CoerceInt96Tz ( tz)
758+ } ) ,
759+ max_predicate_cache_size_opt : global_options. global . max_predicate_cache_size . map ( |size| {
760+ parquet_options:: MaxPredicateCacheSizeOpt :: MaxPredicateCacheSize ( size as u64 )
761+ } ) ,
762+ max_row_group_bytes_opt : global_options. global . max_row_group_bytes . map ( |size| {
763+ parquet_options:: MaxRowGroupBytesOpt :: MaxRowGroupBytes ( size. get ( ) as u64 )
764+ } ) ,
765+ content_defined_chunking : Some ( protobuf:: ParquetCdcOptions {
766+ enabled : global_options. global . content_defined_chunking . enabled ,
767+ min_chunk_size : global_options. global . content_defined_chunking . min_chunk_size as u64 ,
768+ max_chunk_size : global_options. global . content_defined_chunking . max_chunk_size as u64 ,
769+ norm_level : global_options. global . content_defined_chunking . norm_level ,
770+ } ) ,
771+ } ) ,
772+ column_specific_options : column_specific_options. into_iter ( ) . map ( |( column_name, options) | {
773+ protobuf:: ParquetColumnSpecificOptions {
774+ column_name,
775+ options : Some ( protobuf:: ParquetColumnOptions {
776+ bloom_filter_enabled_opt : options. bloom_filter_enabled . map ( |enabled| {
777+ parquet_column_options:: BloomFilterEnabledOpt :: BloomFilterEnabled ( enabled)
778+ } ) ,
779+ encoding_opt : options. encoding . map ( |encoding| {
780+ parquet_column_options:: EncodingOpt :: Encoding ( encoding)
781+ } ) ,
782+ dictionary_enabled_opt : options. dictionary_enabled . map ( |enabled| {
783+ parquet_column_options:: DictionaryEnabledOpt :: DictionaryEnabled ( enabled)
784+ } ) ,
785+ compression_opt : options. compression . map ( |compression| {
786+ parquet_column_options:: CompressionOpt :: Compression ( compression)
787+ } ) ,
788+ statistics_enabled_opt : options. statistics_enabled . map ( |enabled| {
789+ parquet_column_options:: StatisticsEnabledOpt :: StatisticsEnabled ( enabled)
790+ } ) ,
791+ bloom_filter_fpp_opt : options. bloom_filter_fpp . map ( |fpp| {
792+ parquet_column_options:: BloomFilterFppOpt :: BloomFilterFpp ( fpp)
793+ } ) ,
794+ bloom_filter_ndv_opt : options. bloom_filter_ndv . map ( |ndv| {
795+ parquet_column_options:: BloomFilterNdvOpt :: BloomFilterNdv ( ndv)
796+ } ) ,
797+ } )
798+ }
799+ } ) . collect ( ) ,
800+ key_value_metadata : global_options. key_value_metadata
801+ . iter ( )
802+ . filter_map ( |( key, value) | {
803+ value. as_ref ( ) . map ( |v| ( key. clone ( ) , v. clone ( ) ) )
804+ } )
805+ . collect ( ) ,
806+ }
807+ }
808+ }
0 commit comments