@@ -41,10 +41,11 @@ use datafusion_expr::Operator;
4141use crate :: source:: OpenArgs ;
4242use datafusion_physical_expr:: expressions:: { BinaryExpr , Column } ;
4343use datafusion_physical_expr:: projection:: { ProjectionExprs , ProjectionMapping } ;
44+ use datafusion_physical_expr:: utils:: collect_columns;
4445use datafusion_physical_expr:: utils:: reassign_expr_columns;
4546use datafusion_physical_expr:: { EquivalenceProperties , Partitioning , split_conjunction} ;
4647use datafusion_physical_expr_adapter:: PhysicalExprAdapterFactory ;
47- use datafusion_physical_expr_common:: physical_expr:: PhysicalExpr ;
48+ use datafusion_physical_expr_common:: physical_expr:: { PhysicalExpr , is_volatile } ;
4849use datafusion_physical_expr_common:: sort_expr:: { LexOrdering , PhysicalSortExpr } ;
4950use datafusion_physical_plan:: SortOrderPushdownResult ;
5051use datafusion_physical_plan:: coop:: cooperative;
@@ -649,6 +650,45 @@ fn project_output_partitioning(
649650 }
650651}
651652
653+ /// Returns `true` if merging `outer` into `inner` would duplicate a volatile
654+ /// expression; the caller should then decline the merge.
655+ ///
656+ /// `inner` is the scan's current projection and `outer` the projection being
657+ /// pushed into it; merging substitutes each `inner` expression into every
658+ /// `outer` reference to it. If a volatile `inner` expression (e.g. `random()`,
659+ /// `uuid()`) is referenced by `outer`, that one value gets inlined at each site
660+ /// and re-evaluated independently, so references meant to share a single
661+ /// "locked-in" value diverge. This is the volatility guard the physical
662+ /// `ProjectionPushdown` and `FilterPushdown` rules already apply (see
663+ /// `datafusion_physical_expr_common::physical_expr::is_volatile`).
664+ ///
665+ /// A volatile `inner` expression blocks the merge whenever `outer` references
666+ /// it at all: `collect_columns` reports a column at most once per `outer`
667+ /// expression, so a self-duplicating expression such as `r + r` would slip past
668+ /// a stricter `> 1` threshold.
669+ fn would_duplicate_volatile_exprs (
670+ inner : & ProjectionExprs ,
671+ outer : & ProjectionExprs ,
672+ ) -> bool {
673+ let inner_exprs = inner. as_ref ( ) ;
674+ let outer_exprs = outer. as_ref ( ) ;
675+
676+ let mut referenced = vec ! [ false ; inner_exprs. len( ) ] ;
677+ for proj_expr in outer_exprs {
678+ for col in collect_columns ( & proj_expr. expr ) {
679+ let idx = col. index ( ) ;
680+ if idx < referenced. len ( ) {
681+ referenced[ idx] = true ;
682+ }
683+ }
684+ }
685+
686+ referenced
687+ . iter ( )
688+ . enumerate ( )
689+ . any ( |( idx, & referenced) | referenced && is_volatile ( & inner_exprs[ idx] . expr ) )
690+ }
691+
652692impl DataSource for FileScanConfig {
653693 fn open (
654694 & self ,
@@ -945,6 +985,15 @@ impl DataSource for FileScanConfig {
945985 & self ,
946986 projection : & ProjectionExprs ,
947987 ) -> Result < Option < Arc < dyn DataSource > > > {
988+ // Don't merge a projection into the scan if it would inline a volatile
989+ // expression that the outer projection references, which would turn a
990+ // single "locked-in" value (e.g. `random()` aliased in a subquery) into
991+ // multiple independent evaluations. See #23220.
992+ if let Some ( inner) = self . file_source . projection ( )
993+ && would_duplicate_volatile_exprs ( inner, projection)
994+ {
995+ return Ok ( None ) ;
996+ }
948997 match self . file_source . try_pushdown_projection ( projection) ? {
949998 Some ( new_source) => {
950999 let mut new_file_scan_config = self . clone ( ) ;
@@ -3342,4 +3391,162 @@ mod tests {
33423391 ) ;
33433392 Ok ( ( ) )
33443393 }
3394+
3395+ /// Helper: build a `ProjectionExprs` from `(expr, alias)` pairs.
3396+ fn make_projection ( pairs : Vec < ( Arc < dyn PhysicalExpr > , & str ) > ) -> ProjectionExprs {
3397+ ProjectionExprs :: new (
3398+ pairs
3399+ . into_iter ( )
3400+ . map ( |( expr, alias) | ProjectionExpr :: new ( expr, alias) ) ,
3401+ )
3402+ }
3403+
3404+ /// Helper: create a volatile (non-deterministic) function expression,
3405+ /// e.g. `random()`.
3406+ fn make_volatile_expr ( ) -> Arc < dyn PhysicalExpr > {
3407+ use datafusion_common:: config:: ConfigOptions ;
3408+ use datafusion_expr:: ScalarUDF ;
3409+ use datafusion_functions:: math:: random:: RandomFunc ;
3410+ use datafusion_physical_expr:: ScalarFunctionExpr ;
3411+
3412+ Arc :: new ( ScalarFunctionExpr :: new (
3413+ "random" ,
3414+ Arc :: new ( ScalarUDF :: from ( RandomFunc :: new ( ) ) ) ,
3415+ vec ! [ ] ,
3416+ Arc :: new ( Field :: new ( "random" , DataType :: Float64 , false ) ) ,
3417+ Arc :: new ( ConfigOptions :: default ( ) ) ,
3418+ ) )
3419+ }
3420+
3421+ /// Column-only inner projections always merge safely, even when
3422+ /// the outer projection references them multiple times.
3423+ #[ test]
3424+ fn test_would_duplicate_allows_column_only_inner ( ) {
3425+ let col_a: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "a" , 0 ) ) ;
3426+ let col_b: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "b" , 1 ) ) ;
3427+
3428+ let inner =
3429+ make_projection ( vec ! [ ( Arc :: clone( & col_a) , "a" ) , ( Arc :: clone( & col_b) , "b" ) ] ) ;
3430+
3431+ // Outer references col 0 twice
3432+ let outer = make_projection ( vec ! [
3433+ ( Arc :: new( Column :: new( "a" , 0 ) ) , "x" ) ,
3434+ ( Arc :: new( Column :: new( "a" , 0 ) ) , "y" ) ,
3435+ ] ) ;
3436+
3437+ assert ! ( !would_duplicate_volatile_exprs( & inner, & outer) ) ;
3438+ }
3439+
3440+ /// Deterministic computed expressions (arithmetic) referenced multiple
3441+ /// times are allowed to merge — only volatile expressions are protected.
3442+ #[ test]
3443+ fn test_would_duplicate_allows_deterministic_computed_multi_ref ( ) {
3444+ let col_a: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "a" , 0 ) ) ;
3445+ let col_b: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "b" , 1 ) ) ;
3446+ // Inner: [a + b, b] (index 0 is deterministic computed)
3447+ let inner = make_projection ( vec ! [
3448+ (
3449+ Arc :: new( BinaryExpr :: new(
3450+ Arc :: clone( & col_a) ,
3451+ Operator :: Plus ,
3452+ Arc :: clone( & col_b) ,
3453+ ) ) ,
3454+ "sum" ,
3455+ ) ,
3456+ ( Arc :: clone( & col_b) , "b" ) ,
3457+ ] ) ;
3458+
3459+ // Outer references index 0 twice
3460+ let outer = make_projection ( vec ! [
3461+ ( Arc :: new( Column :: new( "sum" , 0 ) ) , "x" ) ,
3462+ ( Arc :: new( Column :: new( "sum" , 0 ) ) , "y" ) ,
3463+ ] ) ;
3464+
3465+ // Deterministic arithmetic → allow merge even though duplicated
3466+ assert ! ( !would_duplicate_volatile_exprs( & inner, & outer) ) ;
3467+ }
3468+
3469+ /// A volatile expression the outer projection does not reference is
3470+ /// safe to merge (it is projected away, not duplicated).
3471+ #[ test]
3472+ fn test_would_duplicate_allows_unreferenced_volatile ( ) {
3473+ let col_a: Arc < dyn PhysicalExpr > = Arc :: new ( Column :: new ( "a" , 0 ) ) ;
3474+ // Inner: [random(), a]
3475+ let inner =
3476+ make_projection ( vec ! [ ( make_volatile_expr( ) , "r" ) , ( Arc :: clone( & col_a) , "a" ) ] ) ;
3477+
3478+ // Outer references only index 1 (the column), not the volatile expr
3479+ let outer = make_projection ( vec ! [ ( Arc :: new( Column :: new( "a" , 1 ) ) , "a" ) ] ) ;
3480+
3481+ assert ! ( !would_duplicate_volatile_exprs( & inner, & outer) ) ;
3482+ }
3483+
3484+ /// A volatile expression referenced multiple times must block merge:
3485+ /// this is the #23220 regression (`random()` aliased then referenced as
3486+ /// `x` and `y`).
3487+ #[ test]
3488+ fn test_would_duplicate_blocks_multi_ref_volatile ( ) {
3489+ // Inner: [random()]
3490+ let inner = make_projection ( vec ! [ ( make_volatile_expr( ) , "r" ) ] ) ;
3491+
3492+ // Outer references index 0 twice
3493+ let outer = make_projection ( vec ! [
3494+ ( Arc :: new( Column :: new( "r" , 0 ) ) , "x" ) ,
3495+ ( Arc :: new( Column :: new( "r" , 0 ) ) , "y" ) ,
3496+ ] ) ;
3497+
3498+ assert ! ( would_duplicate_volatile_exprs( & inner, & outer) ) ;
3499+ }
3500+
3501+ /// A volatile expression referenced by a single outer expression must
3502+ /// also block merge, since `collect_columns` collapses repeated uses
3503+ /// within one expression (e.g. `r + r`) — the `>= 1` threshold.
3504+ #[ test]
3505+ fn test_would_duplicate_blocks_single_expr_self_ref_volatile ( ) {
3506+ // Inner: [random()]
3507+ let inner = make_projection ( vec ! [ ( make_volatile_expr( ) , "r" ) ] ) ;
3508+
3509+ // Outer: [r + r] — one expression, but it duplicates `random()`
3510+ let outer = make_projection ( vec ! [ (
3511+ Arc :: new( BinaryExpr :: new(
3512+ Arc :: new( Column :: new( "r" , 0 ) ) ,
3513+ Operator :: Plus ,
3514+ Arc :: new( Column :: new( "r" , 0 ) ) ,
3515+ ) ) ,
3516+ "x" ,
3517+ ) ] ) ;
3518+
3519+ assert ! ( would_duplicate_volatile_exprs( & inner, & outer) ) ;
3520+ }
3521+
3522+ /// A volatile expression buried inside a larger expression (e.g.
3523+ /// `random() + 1`) is still detected and blocks merge.
3524+ #[ test]
3525+ fn test_would_duplicate_blocks_volatile_nested_in_arithmetic ( ) {
3526+ // Inner: [random() + 1]
3527+ let inner = make_projection ( vec ! [ (
3528+ Arc :: new( BinaryExpr :: new(
3529+ make_volatile_expr( ) ,
3530+ Operator :: Plus ,
3531+ Arc :: new( Literal :: new( ScalarValue :: Float64 ( Some ( 1.0 ) ) ) ) ,
3532+ ) ) ,
3533+ "expr" ,
3534+ ) ] ) ;
3535+
3536+ // Outer references index 0 twice
3537+ let outer = make_projection ( vec ! [
3538+ ( Arc :: new( Column :: new( "expr" , 0 ) ) , "x" ) ,
3539+ ( Arc :: new( Column :: new( "expr" , 0 ) ) , "y" ) ,
3540+ ] ) ;
3541+
3542+ assert ! ( would_duplicate_volatile_exprs( & inner, & outer) ) ;
3543+ }
3544+
3545+ /// Empty projections should not block merging.
3546+ #[ test]
3547+ fn test_would_duplicate_empty_projections ( ) {
3548+ let inner = make_projection ( vec ! [ ] ) ;
3549+ let outer = make_projection ( vec ! [ ] ) ;
3550+ assert ! ( !would_duplicate_volatile_exprs( & inner, & outer) ) ;
3551+ }
33453552}
0 commit comments