2121//! Note: Most traits here need to be marked `Sync + Send` to be
2222//! compliant with the `SendableRecordBatchStream` trait.
2323
24+ mod builder;
25+
2426use std:: collections:: VecDeque ;
2527use std:: pin:: Pin ;
2628use std:: sync:: Arc ;
@@ -29,7 +31,7 @@ use std::task::{Context, Poll};
2931use crate :: PartitionedFile ;
3032use crate :: file_scan_config:: FileScanConfig ;
3133use arrow:: datatypes:: SchemaRef ;
32- use datafusion_common:: error :: Result ;
34+ use datafusion_common:: Result ;
3335use datafusion_execution:: RecordBatchStream ;
3436use datafusion_physical_plan:: metrics:: {
3537 BaselineMetrics , Count , ExecutionPlanMetricsSet , MetricBuilder , MetricCategory , Time ,
@@ -42,6 +44,8 @@ use futures::future::BoxFuture;
4244use futures:: stream:: BoxStream ;
4345use futures:: { FutureExt as _, Stream , StreamExt as _, ready} ;
4446
47+ pub use builder:: FileStreamBuilder ;
48+
4549/// A stream that iterates record batch by record batch, file over file.
4650pub struct FileStream {
4751 /// An iterator over input files.
@@ -66,26 +70,18 @@ pub struct FileStream {
6670
6771impl FileStream {
6872 /// Create a new `FileStream` using the give `FileOpener` to scan underlying files
73+ #[ deprecated( since = "54.0.0" , note = "Use FileStreamBuilder instead" ) ]
6974 pub fn new (
7075 config : & FileScanConfig ,
7176 partition : usize ,
7277 file_opener : Arc < dyn FileOpener > ,
7378 metrics : & ExecutionPlanMetricsSet ,
7479 ) -> Result < Self > {
75- let projected_schema = config. projected_schema ( ) ?;
76-
77- let file_group = config. file_groups [ partition] . clone ( ) ;
78-
79- Ok ( Self {
80- file_iter : file_group. into_inner ( ) . into_iter ( ) . collect ( ) ,
81- projected_schema,
82- remain : config. limit ,
83- file_opener,
84- state : FileStreamState :: Idle ,
85- file_stream_metrics : FileStreamMetrics :: new ( metrics, partition) ,
86- baseline_metrics : BaselineMetrics :: new ( metrics, partition) ,
87- on_error : OnError :: Fail ,
88- } )
80+ FileStreamBuilder :: new ( config)
81+ . with_partition ( partition)
82+ . with_file_opener ( file_opener)
83+ . with_metrics ( metrics)
84+ . build ( )
8985 }
9086
9187 /// Specify the behavior when an error occurs opening or scanning a file
@@ -400,17 +396,17 @@ impl FileStreamMetrics {
400396
401397#[ cfg( test) ]
402398mod tests {
403- use crate :: PartitionedFile ;
404- use crate :: file_scan_config:: FileScanConfigBuilder ;
399+ use crate :: file_scan_config:: { FileScanConfig , FileScanConfigBuilder } ;
405400 use crate :: tests:: make_partition;
401+ use crate :: { PartitionedFile , TableSchema } ;
406402 use datafusion_common:: error:: Result ;
407403 use datafusion_execution:: object_store:: ObjectStoreUrl ;
408404 use datafusion_physical_plan:: metrics:: ExecutionPlanMetricsSet ;
409405 use futures:: { FutureExt as _, StreamExt as _} ;
410406 use std:: sync:: Arc ;
411407 use std:: sync:: atomic:: { AtomicUsize , Ordering } ;
412408
413- use crate :: file_stream:: { FileOpenFuture , FileOpener , FileStream , OnError } ;
409+ use crate :: file_stream:: { FileOpenFuture , FileOpener , FileStreamBuilder , OnError } ;
414410 use crate :: test_util:: MockSource ;
415411 use arrow:: array:: RecordBatch ;
416412 use arrow:: datatypes:: Schema ;
@@ -530,7 +526,7 @@ mod tests {
530526
531527 let on_error = self . on_error ;
532528
533- let table_schema = crate :: table_schema :: TableSchema :: new ( file_schema, vec ! [ ] ) ;
529+ let table_schema = TableSchema :: new ( file_schema, vec ! [ ] ) ;
534530 let config = FileScanConfigBuilder :: new (
535531 ObjectStoreUrl :: parse ( "test:///" ) . unwrap ( ) ,
536532 Arc :: new ( MockSource :: new ( table_schema) ) ,
@@ -539,10 +535,12 @@ mod tests {
539535 . with_limit ( self . limit )
540536 . build ( ) ;
541537 let metrics_set = ExecutionPlanMetricsSet :: new ( ) ;
542- let file_stream =
543- FileStream :: new ( & config, 0 , Arc :: new ( self . opener ) , & metrics_set)
544- . unwrap ( )
545- . with_on_error ( on_error) ;
538+ let file_stream = FileStreamBuilder :: new ( & config)
539+ . with_partition ( 0 )
540+ . with_file_opener ( Arc :: new ( self . opener ) )
541+ . with_metrics ( & metrics_set)
542+ . with_on_error ( on_error)
543+ . build ( ) ?;
546544
547545 file_stream
548546 . collect :: < Vec < _ > > ( )
@@ -563,6 +561,23 @@ mod tests {
563561 . expect ( "error executing stream" )
564562 }
565563
564+ /// Create the smallest valid file scan config for builder validation tests.
565+ fn builder_test_config ( ) -> FileScanConfig {
566+ let table_schema = TableSchema :: new ( Arc :: new ( Schema :: empty ( ) ) , vec ! [ ] ) ;
567+ FileScanConfigBuilder :: new (
568+ ObjectStoreUrl :: parse ( "test:///" ) . unwrap ( ) ,
569+ Arc :: new ( MockSource :: new ( table_schema) ) ,
570+ )
571+ . with_file ( PartitionedFile :: new ( "mock_file" , 10 ) )
572+ . build ( )
573+ }
574+
575+ /// Convenience helper to keep builder error assertions focused on the
576+ /// specific missing or invalid input under test.
577+ fn builder_error ( builder : FileStreamBuilder < ' _ > ) -> String {
578+ builder. build ( ) . err ( ) . unwrap ( ) . to_string ( )
579+ }
580+
566581 #[ tokio:: test]
567582 async fn on_error_opening ( ) -> Result < ( ) > {
568583 let batches = FileStreamTest :: new ( )
@@ -857,4 +872,36 @@ mod tests {
857872
858873 Ok ( ( ) )
859874 }
875+
876+ #[ test]
877+ fn builder_requires_partition_file_opener_and_metrics ( ) {
878+ let config = builder_test_config ( ) ;
879+
880+ let err = builder_error ( FileStreamBuilder :: new ( & config) ) ;
881+ assert ! ( err. contains( "FileStreamBuilder missing required partition" ) ) ;
882+
883+ let err = builder_error ( FileStreamBuilder :: new ( & config) . with_partition ( 0 ) ) ;
884+ assert ! ( err. contains( "FileStreamBuilder missing required file_opener" ) ) ;
885+
886+ let err = builder_error (
887+ FileStreamBuilder :: new ( & config)
888+ . with_partition ( 0 )
889+ . with_file_opener ( Arc :: new ( TestOpener :: default ( ) ) ) ,
890+ ) ;
891+ assert ! ( err. contains( "FileStreamBuilder missing required metrics" ) ) ;
892+ }
893+
894+ #[ test]
895+ fn builder_errors_on_invalid_partition ( ) {
896+ let config = builder_test_config ( ) ;
897+ let metrics = ExecutionPlanMetricsSet :: new ( ) ;
898+
899+ let err = builder_error (
900+ FileStreamBuilder :: new ( & config)
901+ . with_partition ( 1 )
902+ . with_file_opener ( Arc :: new ( TestOpener :: default ( ) ) )
903+ . with_metrics ( & metrics) ,
904+ ) ;
905+ assert ! ( err. contains( "FileStreamBuilder invalid partition index: 1" ) ) ;
906+ }
860907}
0 commit comments