@@ -97,6 +97,8 @@ use datafusion_comet_spark_expr::{
9797 CsvWriteOptions , EvalMode , ListPositionsExpr , SparkArraysZipFunc , SparkBloomFilterVersion ,
9898 SparkListAgg , SparkPercentile , Subquery , SumInteger , ToCsv ,
9999} ;
100+ use datafusion_datasource:: file_scan_config:: FileScanConfig ;
101+ use datafusion_datasource:: source:: DataSourceExec ;
100102use datafusion_datasource:: TableSchema ;
101103use iceberg:: expr:: Bind ;
102104
@@ -3080,23 +3082,25 @@ impl PhysicalPlanner {
30803082
30813083 /// `plan`, declaring `sort_orders` as the order its rows arrive in, unless it already
30823084 /// declares one. Spark's plan knows the order of an input that enters the native plan from
3083- /// the JVM, but the `ScanExec` it arrives through declares none. Only that order is declared:
3084- /// when an operator in between may have moved rows, `plan` is left as it is.
3085+ /// the JVM, and of a scan of sorted files, but the `ScanExec` or the file scan the rows
3086+ /// arrive through declares none. Only that order is declared: when an operator in between may
3087+ /// have moved rows, `plan` is left as it is.
30853088 ///
30863089 /// The serde sends an order only when the streamed side's Comet operators report it, and it
30873090 /// checks that nothing under them moves rows (`mayReportFalseOrder`), while this walk asks
30883091 /// the native operators the same thing through `maintains_input_order`. The two agree as
3089- /// long as a Comet operator that reports an order keeps it natively and says so. One that
3090- /// does not lands here with `plan` unchanged, and the join then moves its unmatched rows
3091- /// after Spark has dropped the sort above it. Nothing here can fall back at that point, so
3092- /// the invariant is kept at the operator: see "Adding a New Operator" in the contributor
3093- /// guide.
3092+ /// long as a Comet operator that reports an order keeps it natively and says so, and a Comet
3093+ /// leaf that reports one is a source this walk knows or declares its order itself, as the
3094+ /// native range does. One that does not lands here with `plan` unchanged, and the join then
3095+ /// moves its unmatched rows after Spark has dropped the sort above it. Nothing here can fall
3096+ /// back at that point, so the invariant is kept at the operator: see "Adding a New Operator"
3097+ /// in the contributor guide.
30943098 fn with_declared_order (
30953099 & self ,
30963100 plan : Arc < dyn ExecutionPlan > ,
30973101 sort_orders : & [ Expr ] ,
30983102 ) -> Result < Arc < dyn ExecutionPlan > , ExecutionError > {
3099- if plan. output_ordering ( ) . is_some ( ) || !Self :: keeps_jvm_input_order ( & plan) {
3103+ if plan. output_ordering ( ) . is_some ( ) || !Self :: keeps_source_order ( & plan) {
31003104 return Ok ( plan) ;
31013105 }
31023106 let exprs = sort_orders
@@ -3109,14 +3113,23 @@ impl PhysicalPlanner {
31093113 }
31103114 }
31113115
3112- /// Whether `plan`'s rows come from an input outside the native plan through operators that
3113- /// keep their input's order, so that the order Spark reports for them still holds. Any other
3114- /// operator may have moved rows, such as the null-aware anti join, whose streamed side Comet
3115- /// builds on, and declaring Spark's order over it would be wrong.
3116- fn keeps_jvm_input_order ( plan : & Arc < dyn ExecutionPlan > ) -> bool {
3116+ /// Whether `plan`'s rows come from a source whose order Spark knows through operators that
3117+ /// keep their input's order, so that the order Spark reports for them still holds. A source is
3118+ /// an input from the JVM, or a scan of files, which reads the files of its one file group one
3119+ /// after the other: Spark reports an order for a file scan only when every partition reads one
3120+ /// sorted file, as for a sorted bucketed table with one file per bucket. Any other operator may
3121+ /// have moved rows, such as the null-aware anti join, whose streamed side Comet builds on, and
3122+ /// declaring Spark's order over it would be wrong.
3123+ fn keeps_source_order ( plan : & Arc < dyn ExecutionPlan > ) -> bool {
31173124 if plan. is :: < ScanExec > ( ) || plan. is :: < ShuffleScanExec > ( ) {
31183125 return true ;
31193126 }
3127+ if let Some ( scan) = plan. downcast_ref :: < DataSourceExec > ( ) {
3128+ return scan
3129+ . data_source ( )
3130+ . downcast_ref :: < FileScanConfig > ( )
3131+ . is_some_and ( |config| config. file_groups . len ( ) == 1 ) ;
3132+ }
31203133 // DataFusion reports every right hash join as keeping its probe order, but a right outer
31213134 // join keeps it only when the probe declares one, and no input on this path declares an
31223135 // order. An inner join emits its matches probe row by probe row, and a right semi, anti or
@@ -3139,7 +3152,7 @@ impl PhysicalPlanner {
31393152 . zip ( plan. maintains_input_order ( ) )
31403153 . filter_map ( |( child, keeps) | keeps. then_some ( child) ) ;
31413154 match ( kept. next ( ) , kept. next ( ) ) {
3142- ( Some ( child) , None ) => Self :: keeps_jvm_input_order ( child) ,
3155+ ( Some ( child) , None ) => Self :: keeps_source_order ( child) ,
31433156 _ => false ,
31443157 }
31453158 }
@@ -6564,6 +6577,26 @@ mod tests {
65646577 ) ;
65656578 }
65666579
6580+ #[ test]
6581+ fn declared_order_reaches_a_scan_of_one_file_group ( ) {
6582+ // A scan of files reads the files of its one file group one after the other, so the order
6583+ // Spark reports for it, a sorted bucketed table's, holds. With more than one file group,
6584+ // DataFusion may hand a file to another partition.
6585+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , true ) ] ) ) ;
6586+ let scan = |groups : usize | -> Arc < dyn ExecutionPlan > {
6587+ let source = Arc :: new ( ParquetSource :: new ( Arc :: clone ( & schema) ) ) as Arc < dyn FileSource > ;
6588+ let file_groups = ( 0 ..groups)
6589+ . map ( |i| FileGroup :: new ( vec ! [ PartitionedFile :: new( format!( "f{i}.parquet" ) , 8 ) ] ) )
6590+ . collect ( ) ;
6591+ let config = FileScanConfigBuilder :: new ( ObjectStoreUrl :: local_filesystem ( ) , source)
6592+ . with_file_groups ( file_groups)
6593+ . build ( ) ;
6594+ Arc :: new ( DataSourceExec :: new ( Arc :: new ( config) ) )
6595+ } ;
6596+ assert ! ( PhysicalPlanner :: keeps_source_order( & scan( 1 ) ) ) ;
6597+ assert ! ( !PhysicalPlanner :: keeps_source_order( & scan( 2 ) ) ) ;
6598+ }
6599+
65676600 #[ tokio:: test]
65686601 async fn spark_plan_dynamic_filter_preserves_metrics ( ) {
65696602 use crate :: execution:: metrics:: utils:: to_native_metric_node;
0 commit comments