@@ -46,6 +46,7 @@ use datafusion_common::{
4646 parsers:: CompressionTypeVariant ,
4747 plan_datafusion_err,
4848 stats:: Precision ,
49+ utils:: usize_from_wire,
4950} ;
5051
5152#[ derive( Debug ) ]
@@ -738,30 +739,23 @@ impl From<protobuf::IntervalUnit> for IntervalUnit {
738739 }
739740}
740741
741- impl From < protobuf:: Constraints > for Constraints {
742- fn from ( constraints : protobuf:: Constraints ) -> Self {
743- Constraints :: new_unverified (
744- constraints
745- . constraints
746- . into_iter ( )
747- . map ( |item| item. into ( ) )
748- . collect ( ) ,
749- )
742+ impl TryFrom < protobuf:: Constraints > for Constraints {
743+ type Error = DataFusionError ;
744+
745+ fn try_from (
746+ constraints : protobuf:: Constraints ,
747+ ) -> datafusion_common:: Result < Self , Self :: Error > {
748+ Self :: try_from ( & constraints)
750749 }
751750}
752751
753- impl From < protobuf:: Constraint > for Constraint {
754- fn from ( value : protobuf:: Constraint ) -> Self {
755- match value. constraint_mode . unwrap ( ) {
756- protobuf:: constraint:: ConstraintMode :: PrimaryKey ( elem) => {
757- Constraint :: PrimaryKey (
758- elem. indices . into_iter ( ) . map ( |item| item as usize ) . collect ( ) ,
759- )
760- }
761- protobuf:: constraint:: ConstraintMode :: Unique ( elem) => Constraint :: Unique (
762- elem. indices . into_iter ( ) . map ( |item| item as usize ) . collect ( ) ,
763- ) ,
764- }
752+ impl TryFrom < protobuf:: Constraint > for Constraint {
753+ type Error = DataFusionError ;
754+
755+ fn try_from (
756+ value : protobuf:: Constraint ,
757+ ) -> datafusion_common:: Result < Self , Self :: Error > {
758+ Constraint :: try_from ( & value)
765759 }
766760}
767761
@@ -884,20 +878,28 @@ impl From<protobuf::JoinSide> for JoinSide {
884878 }
885879}
886880
887- impl From < & protobuf:: Constraint > for Constraint {
888- fn from ( value : & protobuf:: Constraint ) -> Self {
881+ impl TryFrom < & protobuf:: Constraint > for Constraint {
882+ type Error = DataFusionError ;
883+
884+ fn try_from (
885+ value : & protobuf:: Constraint ,
886+ ) -> datafusion_common:: Result < Self , Self :: Error > {
887+ let decode_indices = |indices : & [ u64 ] | {
888+ indices
889+ . iter ( )
890+ . map ( |& index| usize_from_wire ( index, "Constraint" , "indices" ) )
891+ . collect :: < datafusion_common:: Result < Vec < _ > > > ( )
892+ } ;
889893 match & value. constraint_mode {
890894 Some ( protobuf:: constraint:: ConstraintMode :: PrimaryKey ( elem) ) => {
891- Constraint :: PrimaryKey (
892- elem. indices . iter ( ) . map ( |& item| item as usize ) . collect ( ) ,
893- )
895+ Ok ( Constraint :: PrimaryKey ( decode_indices ( & elem. indices ) ?) )
894896 }
895897 Some ( protobuf:: constraint:: ConstraintMode :: Unique ( elem) ) => {
896- Constraint :: Unique (
897- elem. indices . iter ( ) . map ( |& item| item as usize ) . collect ( ) ,
898- )
898+ Ok ( Constraint :: Unique ( decode_indices ( & elem. indices ) ?) )
899899 }
900- None => panic ! ( "constraint_mode not set" ) ,
900+ None => Err ( plan_datafusion_err ! (
901+ "Constraint: missing required field 'constraint_mode'"
902+ ) ) ,
901903 }
902904 }
903905}
@@ -912,8 +914,8 @@ impl TryFrom<&protobuf::Constraints> for Constraints {
912914 constraints
913915 . constraints
914916 . iter ( )
915- . map ( |item| item . into ( ) )
916- . collect ( ) ,
917+ . map ( Constraint :: try_from )
918+ . collect :: < datafusion_common :: Result < Vec < _ > > > ( ) ? ,
917919 ) )
918920 }
919921}
@@ -1016,7 +1018,10 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {
10161018 newlines_in_values : proto_opts. newlines_in_values . first ( ) . map ( |h| * h != 0 ) ,
10171019 compression : proto_opts. compression ( ) . into ( ) ,
10181020 compression_level : proto_opts. compression_level ,
1019- schema_infer_max_rec : proto_opts. schema_infer_max_rec . map ( |h| h as usize ) ,
1021+ schema_infer_max_rec : proto_opts
1022+ . schema_infer_max_rec
1023+ . map ( |v| usize_from_wire ( v, "CsvOptions" , "schema_infer_max_rec" ) )
1024+ . transpose ( ) ?,
10201025 date_format : ( !proto_opts. date_format . is_empty ( ) )
10211026 . then ( || proto_opts. date_format . clone ( ) ) ,
10221027 datetime_format : ( !proto_opts. datetime_format . is_empty ( ) )
@@ -1052,21 +1057,28 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
10521057 fn try_from (
10531058 value : & protobuf:: ParquetOptions ,
10541059 ) -> datafusion_common:: Result < Self , Self :: Error > {
1060+ let to_usize =
1061+ |value : u64 , field : & str | usize_from_wire ( value, "ParquetOptions" , field) ;
10551062 Ok ( ParquetOptions {
10561063 enable_page_index : value. enable_page_index ,
10571064 pruning : value. pruning ,
10581065 skip_metadata : value. skip_metadata ,
10591066 metadata_size_hint : value
10601067 . metadata_size_hint_opt
10611068 . map ( |opt| match opt {
1062- protobuf:: parquet_options:: MetadataSizeHintOpt :: MetadataSizeHint ( v) => Some ( v as usize ) ,
1069+ protobuf:: parquet_options:: MetadataSizeHintOpt :: MetadataSizeHint ( v) => {
1070+ to_usize ( v, "metadata_size_hint" )
1071+ }
10631072 } )
1064- . unwrap_or ( None ) ,
1073+ . transpose ( ) ? ,
10651074 pushdown_filters : value. pushdown_filters ,
10661075 reorder_filters : value. reorder_filters ,
10671076 force_filter_selections : value. force_filter_selections ,
1068- data_pagesize_limit : value. data_pagesize_limit as usize ,
1069- write_batch_size : value. write_batch_size as usize ,
1077+ data_pagesize_limit : to_usize (
1078+ value. data_pagesize_limit ,
1079+ "data_pagesize_limit" ,
1080+ ) ?,
1081+ write_batch_size : to_usize ( value. write_batch_size , "write_batch_size" ) ?,
10701082 writer_version : value. writer_version . parse ( ) . map_err ( |e| {
10711083 DataFusionError :: Internal ( format ! ( "Failed to parse writer_version: {e}" ) )
10721084 } ) ?,
@@ -1075,29 +1087,39 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
10751087 } ) . unwrap_or ( None ) ,
10761088 dictionary_enabled : value. dictionary_enabled_opt . as_ref ( ) . map ( |protobuf:: parquet_options:: DictionaryEnabledOpt :: DictionaryEnabled ( v) | * v) ,
10771089 // Continuing from where we left off in the TryFrom implementation
1078- dictionary_page_size_limit : value. dictionary_page_size_limit as usize ,
1090+ dictionary_page_size_limit : to_usize (
1091+ value. dictionary_page_size_limit ,
1092+ "dictionary_page_size_limit" ,
1093+ ) ?,
10791094 statistics_enabled : value
10801095 . statistics_enabled_opt . clone ( )
10811096 . map ( |opt| match opt {
10821097 protobuf:: parquet_options:: StatisticsEnabledOpt :: StatisticsEnabled ( v) => Some ( v) ,
10831098 } )
10841099 . unwrap_or ( None ) ,
1085- max_row_group_size : value. max_row_group_size as usize ,
1086- max_in_list_size : value. max_in_list_size as usize ,
1100+ max_row_group_size : to_usize ( value. max_row_group_size , "max_row_group_size" ) ? ,
1101+ max_in_list_size : to_usize ( value. max_in_list_size , "max_in_list_size" ) ? ,
10871102 created_by : value. created_by . clone ( ) ,
10881103 column_index_truncate_length : value
10891104 . column_index_truncate_length_opt . as_ref ( )
10901105 . map ( |opt| match opt {
1091- protobuf:: parquet_options:: ColumnIndexTruncateLengthOpt :: ColumnIndexTruncateLength ( v) => Some ( * v as usize ) ,
1106+ protobuf:: parquet_options:: ColumnIndexTruncateLengthOpt :: ColumnIndexTruncateLength ( v) => {
1107+ to_usize ( * v, "column_index_truncate_length" )
1108+ }
10921109 } )
1093- . unwrap_or ( None ) ,
1110+ . transpose ( ) ? ,
10941111 statistics_truncate_length : value
10951112 . statistics_truncate_length_opt . as_ref ( )
10961113 . map ( |opt| match opt {
1097- protobuf:: parquet_options:: StatisticsTruncateLengthOpt :: StatisticsTruncateLength ( v) => Some ( * v as usize ) ,
1114+ protobuf:: parquet_options:: StatisticsTruncateLengthOpt :: StatisticsTruncateLength ( v) => {
1115+ to_usize ( * v, "statistics_truncate_length" )
1116+ }
10981117 } )
1099- . unwrap_or ( None ) ,
1100- data_page_row_count_limit : value. data_page_row_count_limit as usize ,
1118+ . transpose ( ) ?,
1119+ data_page_row_count_limit : to_usize (
1120+ value. data_page_row_count_limit ,
1121+ "data_page_row_count_limit" ,
1122+ ) ?,
11011123 encoding : value
11021124 . encoding_opt . clone ( )
11031125 . map ( |opt| match opt {
@@ -1119,8 +1141,14 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
11191141 } )
11201142 . unwrap_or ( None ) ,
11211143 allow_single_file_parallelism : value. allow_single_file_parallelism ,
1122- maximum_parallel_row_group_writers : value. maximum_parallel_row_group_writers as usize ,
1123- maximum_buffered_record_batches_per_stream : value. maximum_buffered_record_batches_per_stream as usize ,
1144+ maximum_parallel_row_group_writers : to_usize (
1145+ value. maximum_parallel_row_group_writers ,
1146+ "maximum_parallel_row_group_writers" ,
1147+ ) ?,
1148+ maximum_buffered_record_batches_per_stream : to_usize (
1149+ value. maximum_buffered_record_batches_per_stream ,
1150+ "maximum_buffered_record_batches_per_stream" ,
1151+ ) ?,
11241152 schema_force_view_types : value. schema_force_view_types ,
11251153 binary_as_string : value. binary_as_string ,
11261154 coerce_int96 : value. coerce_int96_opt . clone ( ) . map ( |opt| match opt {
@@ -1131,24 +1159,34 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
11311159 } ) . unwrap_or ( None ) ,
11321160 skip_arrow_metadata : value. skip_arrow_metadata ,
11331161 max_predicate_cache_size : value. max_predicate_cache_size_opt . map ( |opt| match opt {
1134- protobuf:: parquet_options:: MaxPredicateCacheSizeOpt :: MaxPredicateCacheSize ( v) => Some ( v as usize ) ,
1135- } ) . unwrap_or ( None ) ,
1136- max_row_group_bytes : value. max_row_group_bytes_opt . and_then ( |opt| match opt {
1137- protobuf:: parquet_options:: MaxRowGroupBytesOpt :: MaxRowGroupBytes ( v) => MaxRowGroupBytes :: try_new ( v as usize ) . ok ( ) ,
1138- } ) ,
1139- content_defined_chunking : value. content_defined_chunking . map ( ParquetCdcOptions :: from) . unwrap_or_default ( ) ,
1162+ protobuf:: parquet_options:: MaxPredicateCacheSizeOpt :: MaxPredicateCacheSize ( v) => {
1163+ to_usize ( v, "max_predicate_cache_size" )
1164+ }
1165+ } ) . transpose ( ) ?,
1166+ max_row_group_bytes : value. max_row_group_bytes_opt . map ( |opt| match opt {
1167+ protobuf:: parquet_options:: MaxRowGroupBytesOpt :: MaxRowGroupBytes ( v) => {
1168+ MaxRowGroupBytes :: try_new ( to_usize ( v, "max_row_group_bytes" ) ?)
1169+ }
1170+ } ) . transpose ( ) ?,
1171+ content_defined_chunking : value. content_defined_chunking . map ( ParquetCdcOptions :: try_from) . transpose ( ) ?. unwrap_or_default ( ) ,
11401172 } )
11411173 }
11421174}
11431175
1144- impl From < protobuf:: ParquetCdcOptions > for ParquetCdcOptions {
1145- fn from ( value : protobuf:: ParquetCdcOptions ) -> Self {
1146- ParquetCdcOptions {
1176+ impl TryFrom < protobuf:: ParquetCdcOptions > for ParquetCdcOptions {
1177+ type Error = DataFusionError ;
1178+
1179+ fn try_from (
1180+ value : protobuf:: ParquetCdcOptions ,
1181+ ) -> datafusion_common:: Result < Self , Self :: Error > {
1182+ let to_usize =
1183+ |value : u64 , field : & str | usize_from_wire ( value, "ParquetCdcOptions" , field) ;
1184+ Ok ( ParquetCdcOptions {
11471185 enabled : value. enabled ,
1148- min_chunk_size : value. min_chunk_size as usize ,
1149- max_chunk_size : value. max_chunk_size as usize ,
1186+ min_chunk_size : to_usize ( value. min_chunk_size , "min_chunk_size" ) ? ,
1187+ max_chunk_size : to_usize ( value. max_chunk_size , "max_chunk_size" ) ? ,
11501188 norm_level : value. norm_level ,
1151- }
1189+ } )
11521190 }
11531191}
11541192
@@ -1214,9 +1252,9 @@ impl TryFrom<&protobuf::TableParquetOptions> for TableParquetOptions {
12141252 global : value
12151253 . global
12161254 . as_ref ( )
1217- . map ( |v| v . try_into ( ) )
1218- . unwrap ( )
1219- . unwrap ( ) ,
1255+ . map ( ParquetOptions :: try_from )
1256+ . transpose ( ) ?
1257+ . unwrap_or_default ( ) ,
12201258 column_specific_options,
12211259 ..Default :: default ( )
12221260 } ;
@@ -1234,7 +1272,10 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {
12341272 Ok ( JsonOptions {
12351273 compression : compression. into ( ) ,
12361274 compression_level : proto_opts. compression_level ,
1237- schema_infer_max_rec : proto_opts. schema_infer_max_rec . map ( |h| h as usize ) ,
1275+ schema_infer_max_rec : proto_opts
1276+ . schema_infer_max_rec
1277+ . map ( |v| usize_from_wire ( v, "JsonOptions" , "schema_infer_max_rec" ) )
1278+ . transpose ( ) ?,
12381279 newline_delimited : proto_opts. newline_delimited . unwrap_or ( true ) ,
12391280 } )
12401281 }
@@ -1338,6 +1379,72 @@ mod tests {
13381379 MaxRowGroupBytes , ParquetCdcOptions , ParquetOptions , TableParquetOptions ,
13391380 } ;
13401381
1382+ #[ test]
1383+ fn constraint_requires_mode ( ) {
1384+ let err = datafusion_common:: Constraint :: try_from (
1385+ & crate :: protobuf_common:: Constraint {
1386+ constraint_mode : None ,
1387+ } ,
1388+ )
1389+ . unwrap_err ( ) ;
1390+ assert ! (
1391+ err. to_string( )
1392+ . contains( "Constraint: missing required field 'constraint_mode'" )
1393+ ) ;
1394+ }
1395+
1396+ #[ test]
1397+ fn table_parquet_options_defaults_missing_global ( ) {
1398+ let recovered = TableParquetOptions :: try_from (
1399+ & crate :: protobuf_common:: TableParquetOptions :: default ( ) ,
1400+ )
1401+ . expect ( "missing global should use defaults" ) ;
1402+
1403+ assert_eq ! ( recovered, TableParquetOptions :: default ( ) ) ;
1404+ }
1405+
1406+ #[ test]
1407+ fn table_parquet_options_checks_nested_usize_conversion ( ) {
1408+ let proto = crate :: protobuf_common:: TableParquetOptions {
1409+ global : Some ( crate :: protobuf_common:: ParquetOptions {
1410+ data_pagesize_limit : u64:: MAX ,
1411+ writer_version : "1.0" . to_string ( ) ,
1412+ ..Default :: default ( )
1413+ } ) ,
1414+ ..Default :: default ( )
1415+ } ;
1416+ let recovered = TableParquetOptions :: try_from ( & proto) ;
1417+
1418+ if usize:: BITS >= u64:: BITS {
1419+ assert_eq ! ( recovered. unwrap( ) . global. data_pagesize_limit, usize :: MAX ) ;
1420+ } else {
1421+ assert_eq ! (
1422+ recovered. unwrap_err( ) . strip_backtrace( ) ,
1423+ "Error during planning: ParquetOptions: data_pagesize_limit wire value 18446744073709551615 is out of range for usize"
1424+ ) ;
1425+ }
1426+ }
1427+
1428+ #[ test]
1429+ fn table_parquet_options_propagates_nested_conversion_error ( ) {
1430+ let proto = crate :: protobuf_common:: TableParquetOptions {
1431+ global : Some ( crate :: protobuf_common:: ParquetOptions {
1432+ writer_version : "1.0" . to_string ( ) ,
1433+ max_row_group_bytes_opt : Some (
1434+ crate :: protobuf_common:: parquet_options:: MaxRowGroupBytesOpt :: MaxRowGroupBytes ( 0 ) ,
1435+ ) ,
1436+ ..Default :: default ( )
1437+ } ) ,
1438+ ..Default :: default ( )
1439+ } ;
1440+
1441+ let err = TableParquetOptions :: try_from ( & proto) . unwrap_err ( ) ;
1442+ assert_eq ! (
1443+ err. strip_backtrace( ) ,
1444+ "Invalid or Unsupported Configuration: max_row_group_bytes must be greater than 0"
1445+ ) ;
1446+ }
1447+
13411448 fn parquet_options_proto_round_trip ( opts : ParquetOptions ) -> ParquetOptions {
13421449 let proto: crate :: protobuf_common:: ParquetOptions =
13431450 ( & opts) . try_into ( ) . expect ( "to_proto" ) ;
0 commit comments