@@ -688,3 +688,129 @@ pub fn statistics_from_parquet_meta_calc(
688688) -> Result < Statistics > {
689689 DFParquetMetadata :: statistics_from_parquet_metadata ( metadata, & table_schema)
690690}
691+
692+ #[ cfg( feature = "proto" ) ]
693+ use datafusion_proto_models:: protobuf:: { self , parquet_column_options, parquet_options} ;
694+
695+ /// Encode a [`ParquetFormatFactory`]'s options as their protobuf form.
696+ ///
697+ /// The reverse direction is `TryFrom<&protobuf::TableParquetOptions> for
698+ /// TableParquetOptions` in `datafusion-proto-models`: `TableParquetOptions` is
699+ /// a `datafusion-common` type, so that half cannot live here.
700+ #[ cfg( feature = "proto" ) ]
701+ impl From < & ParquetFormatFactory > for protobuf:: TableParquetOptions {
702+ fn from ( factory : & ParquetFormatFactory ) -> Self {
703+ let global_options = if let Some ( ref options) = factory. options {
704+ options. clone ( )
705+ } else {
706+ return protobuf:: TableParquetOptions :: default ( ) ;
707+ } ;
708+
709+ let column_specific_options = global_options. column_specific_options ;
710+ protobuf:: TableParquetOptions {
711+ global : Some ( protobuf:: ParquetOptions {
712+ enable_page_index : global_options. global . enable_page_index ,
713+ pruning : global_options. global . pruning ,
714+ skip_metadata : global_options. global . skip_metadata ,
715+ metadata_size_hint_opt : global_options. global . metadata_size_hint . map ( |size| {
716+ parquet_options:: MetadataSizeHintOpt :: MetadataSizeHint ( size as u64 )
717+ } ) ,
718+ pushdown_filters : global_options. global . pushdown_filters ,
719+ reorder_filters : global_options. global . reorder_filters ,
720+ force_filter_selections : global_options. global . force_filter_selections ,
721+ data_pagesize_limit : global_options. global . data_pagesize_limit as u64 ,
722+ write_batch_size : global_options. global . write_batch_size as u64 ,
723+ writer_version : global_options. global . writer_version . to_string ( ) ,
724+ compression_opt : global_options. global . compression . map ( |compression| {
725+ parquet_options:: CompressionOpt :: Compression ( compression)
726+ } ) ,
727+ dictionary_enabled_opt : global_options. global . dictionary_enabled . map ( |enabled| {
728+ parquet_options:: DictionaryEnabledOpt :: DictionaryEnabled ( enabled)
729+ } ) ,
730+ dictionary_page_size_limit : global_options. global . dictionary_page_size_limit as u64 ,
731+ statistics_enabled_opt : global_options. global . statistics_enabled . map ( |enabled| {
732+ parquet_options:: StatisticsEnabledOpt :: StatisticsEnabled ( enabled)
733+ } ) ,
734+ max_row_group_size : global_options. global . max_row_group_size as u64 ,
735+ max_in_list_size : global_options. global . max_in_list_size as u64 ,
736+ created_by : global_options. global . created_by . clone ( ) ,
737+ column_index_truncate_length_opt : global_options. global . column_index_truncate_length . map ( |length| {
738+ parquet_options:: ColumnIndexTruncateLengthOpt :: ColumnIndexTruncateLength ( length as u64 )
739+ } ) ,
740+ statistics_truncate_length_opt : global_options. global . statistics_truncate_length . map ( |length| {
741+ parquet_options:: StatisticsTruncateLengthOpt :: StatisticsTruncateLength ( length as u64 )
742+ } ) ,
743+ data_page_row_count_limit : global_options. global . data_page_row_count_limit as u64 ,
744+ encoding_opt : global_options. global . encoding . map ( |encoding| {
745+ parquet_options:: EncodingOpt :: Encoding ( encoding)
746+ } ) ,
747+ bloom_filter_on_read : global_options. global . bloom_filter_on_read ,
748+ bloom_filter_on_write : global_options. global . bloom_filter_on_write ,
749+ bloom_filter_fpp_opt : global_options. global . bloom_filter_fpp . map ( |fpp| {
750+ parquet_options:: BloomFilterFppOpt :: BloomFilterFpp ( fpp)
751+ } ) ,
752+ bloom_filter_ndv_opt : global_options. global . bloom_filter_ndv . map ( |ndv| {
753+ parquet_options:: BloomFilterNdvOpt :: BloomFilterNdv ( ndv)
754+ } ) ,
755+ allow_single_file_parallelism : global_options. global . allow_single_file_parallelism ,
756+ maximum_parallel_row_group_writers : global_options. global . maximum_parallel_row_group_writers as u64 ,
757+ maximum_buffered_record_batches_per_stream : global_options. global . maximum_buffered_record_batches_per_stream as u64 ,
758+ schema_force_view_types : global_options. global . schema_force_view_types ,
759+ binary_as_string : global_options. global . binary_as_string ,
760+ skip_arrow_metadata : global_options. global . skip_arrow_metadata ,
761+ coerce_int96_opt : global_options. global . coerce_int96 . map ( |compression| {
762+ parquet_options:: CoerceInt96Opt :: CoerceInt96 ( compression)
763+ } ) ,
764+ coerce_int96_tz_opt : global_options. global . coerce_int96_tz . map ( |tz| {
765+ parquet_options:: CoerceInt96TzOpt :: CoerceInt96Tz ( tz)
766+ } ) ,
767+ max_predicate_cache_size_opt : global_options. global . max_predicate_cache_size . map ( |size| {
768+ parquet_options:: MaxPredicateCacheSizeOpt :: MaxPredicateCacheSize ( size as u64 )
769+ } ) ,
770+ max_row_group_bytes_opt : global_options. global . max_row_group_bytes . map ( |size| {
771+ parquet_options:: MaxRowGroupBytesOpt :: MaxRowGroupBytes ( size. get ( ) as u64 )
772+ } ) ,
773+ content_defined_chunking : Some ( protobuf:: ParquetCdcOptions {
774+ enabled : global_options. global . content_defined_chunking . enabled ,
775+ min_chunk_size : global_options. global . content_defined_chunking . min_chunk_size as u64 ,
776+ max_chunk_size : global_options. global . content_defined_chunking . max_chunk_size as u64 ,
777+ norm_level : global_options. global . content_defined_chunking . norm_level ,
778+ } ) ,
779+ } ) ,
780+ column_specific_options : column_specific_options. into_iter ( ) . map ( |( column_name, options) | {
781+ protobuf:: ParquetColumnSpecificOptions {
782+ column_name,
783+ options : Some ( protobuf:: ParquetColumnOptions {
784+ bloom_filter_enabled_opt : options. bloom_filter_enabled . map ( |enabled| {
785+ parquet_column_options:: BloomFilterEnabledOpt :: BloomFilterEnabled ( enabled)
786+ } ) ,
787+ encoding_opt : options. encoding . map ( |encoding| {
788+ parquet_column_options:: EncodingOpt :: Encoding ( encoding)
789+ } ) ,
790+ dictionary_enabled_opt : options. dictionary_enabled . map ( |enabled| {
791+ parquet_column_options:: DictionaryEnabledOpt :: DictionaryEnabled ( enabled)
792+ } ) ,
793+ compression_opt : options. compression . map ( |compression| {
794+ parquet_column_options:: CompressionOpt :: Compression ( compression)
795+ } ) ,
796+ statistics_enabled_opt : options. statistics_enabled . map ( |enabled| {
797+ parquet_column_options:: StatisticsEnabledOpt :: StatisticsEnabled ( enabled)
798+ } ) ,
799+ bloom_filter_fpp_opt : options. bloom_filter_fpp . map ( |fpp| {
800+ parquet_column_options:: BloomFilterFppOpt :: BloomFilterFpp ( fpp)
801+ } ) ,
802+ bloom_filter_ndv_opt : options. bloom_filter_ndv . map ( |ndv| {
803+ parquet_column_options:: BloomFilterNdvOpt :: BloomFilterNdv ( ndv)
804+ } ) ,
805+ } )
806+ }
807+ } ) . collect ( ) ,
808+ key_value_metadata : global_options. key_value_metadata
809+ . iter ( )
810+ . filter_map ( |( key, value) | {
811+ value. as_ref ( ) . map ( |v| ( key. clone ( ) , v. clone ( ) ) )
812+ } )
813+ . collect ( ) ,
814+ }
815+ }
816+ }
0 commit comments