@@ -23,16 +23,10 @@ use arrow::array::RecordBatch;
2323use arrow:: compute:: SortOptions ;
2424use arrow:: datatypes:: { Field , Schema } ;
2525use arrow:: ipc:: reader:: StreamReader ;
26- use datafusion_common:: { DataFusionError , Result , internal_datafusion_err, not_impl_err} ;
26+ use datafusion_common:: { Result , internal_datafusion_err, not_impl_err} ;
27+ use datafusion_datasource:: TableSchema ;
2728use datafusion_datasource:: file:: FileSource ;
28- use datafusion_datasource:: file_groups:: FileGroup ;
2929use datafusion_datasource:: file_scan_config:: FileScanConfig ;
30- use datafusion_datasource:: file_sink_config:: FileSinkConfig ;
31- use datafusion_datasource:: { FileRange , PartitionedFile , TableSchema } ;
32- use datafusion_datasource_csv:: file_format:: CsvSink ;
33- use datafusion_datasource_json:: file_format:: JsonSink ;
34- #[ cfg( feature = "parquet" ) ]
35- use datafusion_datasource_parquet:: file_format:: ParquetSink ;
3630use datafusion_execution:: { FunctionRegistry , TaskContext } ;
3731use datafusion_expr:: WindowFunctionDefinition ;
3832use datafusion_physical_expr:: expressions:: { LambdaExpr , LambdaVariable } ;
@@ -467,66 +461,6 @@ pub fn parse_record_batches(buf: &[u8]) -> Result<Vec<RecordBatch>> {
467461 Ok ( batches)
468462}
469463
470- /// Thin shim over `TryFrom<&protobuf::PartitionedFile>`, which owns the wire logic.
471- impl TryFromProto < & protobuf:: PartitionedFile > for PartitionedFile {
472- type Error = DataFusionError ;
473-
474- fn try_from_proto ( val : & protobuf:: PartitionedFile ) -> Result < Self , Self :: Error > {
475- PartitionedFile :: try_from ( val)
476- }
477- }
478-
479- /// Thin shim over `TryFrom<&protobuf::FileRange>`, which owns the wire logic.
480- impl TryFromProto < & protobuf:: FileRange > for FileRange {
481- type Error = DataFusionError ;
482-
483- fn try_from_proto ( value : & protobuf:: FileRange ) -> Result < Self , Self :: Error > {
484- FileRange :: try_from ( value)
485- }
486- }
487-
488- /// Thin shim over `TryFrom<&protobuf::FileGroup>`, which owns the wire logic.
489- impl TryFromProto < & protobuf:: FileGroup > for FileGroup {
490- type Error = DataFusionError ;
491-
492- fn try_from_proto ( val : & protobuf:: FileGroup ) -> Result < Self , Self :: Error > {
493- FileGroup :: try_from ( val)
494- }
495- }
496-
497- impl TryFromProto < & protobuf:: JsonSink > for JsonSink {
498- type Error = DataFusionError ;
499-
500- fn try_from_proto ( value : & protobuf:: JsonSink ) -> Result < Self , Self :: Error > {
501- Self :: try_from ( value)
502- }
503- }
504-
505- #[ cfg( feature = "parquet" ) ]
506- impl TryFromProto < & protobuf:: ParquetSink > for ParquetSink {
507- type Error = DataFusionError ;
508-
509- fn try_from_proto ( value : & protobuf:: ParquetSink ) -> Result < Self , Self :: Error > {
510- Self :: try_from ( value)
511- }
512- }
513-
514- impl TryFromProto < & protobuf:: CsvSink > for CsvSink {
515- type Error = DataFusionError ;
516-
517- fn try_from_proto ( value : & protobuf:: CsvSink ) -> Result < Self , Self :: Error > {
518- Self :: try_from ( value)
519- }
520- }
521-
522- impl TryFromProto < & protobuf:: FileSinkConfig > for FileSinkConfig {
523- type Error = DataFusionError ;
524-
525- fn try_from_proto ( conf : & protobuf:: FileSinkConfig ) -> Result < Self , Self :: Error > {
526- conf. try_into ( )
527- }
528- }
529-
530464/// Concrete [`PhysicalExprDecode`] driver that backs
531465/// [`PhysicalExprDecodeCtx`] inside `parse_physical_expr_with_converter`.
532466///
@@ -553,98 +487,3 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD
553487 . proto_to_physical_expr ( node, schema, self . ctx )
554488 }
555489}
556-
557- #[ cfg( test) ]
558- mod tests {
559- use super :: * ;
560- use arrow:: datatypes:: { DataType , Field , Schema } ;
561- use chrono:: { TimeZone , Utc } ;
562- use datafusion_common:: ScalarValue ;
563- use object_store:: ObjectMeta ;
564- use object_store:: path:: Path ;
565-
566- #[ test]
567- fn partitioned_file_path_roundtrip_percent_encoded ( ) {
568- let path_str = "foo/foo%2Fbar/baz%252Fqux" ;
569- let pf = PartitionedFile :: new_from_meta ( ObjectMeta {
570- location : Path :: parse ( path_str) . unwrap ( ) ,
571- last_modified : Utc . timestamp_nanos ( 1_000 ) ,
572- size : 42 ,
573- e_tag : None ,
574- version : None ,
575- } ) ;
576-
577- let proto = protobuf:: PartitionedFile :: try_from_proto ( & pf) . unwrap ( ) ;
578- assert_eq ! ( proto. path, path_str) ;
579-
580- let pf2 = PartitionedFile :: try_from_proto ( & proto) . unwrap ( ) ;
581- assert_eq ! ( pf2. object_meta. location. as_ref( ) , path_str) ;
582- assert_eq ! ( pf2. object_meta. location, pf. object_meta. location) ;
583- assert_eq ! ( pf2. object_meta. size, pf. object_meta. size) ;
584- assert_eq ! ( pf2. object_meta. last_modified, pf. object_meta. last_modified) ;
585- }
586-
587- #[ test]
588- fn partitioned_file_arrow_schema_roundtrip ( ) {
589- use std:: collections:: HashMap ;
590-
591- let arrow_schema = Arc :: new ( Schema :: new_with_metadata (
592- vec ! [
593- Field :: new( "id" , DataType :: Int64 , false ) ,
594- Field :: new( "value" , DataType :: Utf8 , true ) . with_metadata( HashMap :: from( [
595- ( "field_meta" . to_string( ) , "field_value" . to_string( ) ) ,
596- ] ) ) ,
597- ] ,
598- HashMap :: from ( [ ( "schema_meta" . to_string ( ) , "schema_value" . to_string ( ) ) ] ) ,
599- ) ) ;
600- let pf = PartitionedFile :: new ( "foo/bar.parquet" , 10 )
601- . with_arrow_schema ( Arc :: clone ( & arrow_schema) ) ;
602-
603- let proto = protobuf:: PartitionedFile :: try_from_proto ( & pf) . unwrap ( ) ;
604- assert ! ( proto. arrow_schema. is_some( ) ) ;
605-
606- let decoded = PartitionedFile :: try_from_proto ( & proto) . unwrap ( ) ;
607- assert_eq ! (
608- decoded. arrow_schema. as_ref( ) . map( |s| s. as_ref( ) ) ,
609- Some ( arrow_schema. as_ref( ) )
610- ) ;
611- }
612-
613- #[ test]
614- fn partitioned_file_statistics_roundtrip_with_partition_values ( ) {
615- use datafusion_common:: Statistics ;
616- let file_schema = Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , true ) ] ) ;
617- let pf = PartitionedFile :: new ( "foo/bar.parquet" , 1234 )
618- . with_partition_values ( vec ! [ ScalarValue :: from( "2024-01-01" ) ] )
619- . with_statistics ( Arc :: new ( Statistics :: new_unknown ( & file_schema) ) ) ;
620-
621- // `statistics` covers the full table schema: file columns followed by one
622- // entry per partition column.
623- let expected_len = file_schema. fields ( ) . len ( ) + pf. partition_values . len ( ) ;
624- assert_eq ! (
625- pf. statistics. as_ref( ) . unwrap( ) . column_statistics. len( ) ,
626- expected_len
627- ) ;
628-
629- let proto = protobuf:: PartitionedFile :: try_from_proto ( & pf) . unwrap ( ) ;
630- let decoded = PartitionedFile :: try_from_proto ( & proto) . unwrap ( ) ;
631-
632- assert_eq ! ( decoded. statistics, pf. statistics) ;
633- }
634-
635- #[ test]
636- fn partitioned_file_from_proto_invalid_path ( ) {
637- let proto = protobuf:: PartitionedFile {
638- arrow_schema : None ,
639- path : "foo//bar" . to_string ( ) ,
640- size : 1 ,
641- last_modified_ns : 0 ,
642- partition_values : vec ! [ ] ,
643- range : None ,
644- statistics : None ,
645- } ;
646-
647- let err = PartitionedFile :: try_from_proto ( & proto) . unwrap_err ( ) ;
648- assert ! ( err. to_string( ) . contains( "Invalid object_store path" ) ) ;
649- }
650- }
0 commit comments