Skip to content

Commit c62b344

Browse files
committed
fix: don't duplicate volatile expressions when pushing projection into file scan
A volatile expression (e.g. `random()`, `uuid()`) aliased once in a subquery and referenced multiple times must be evaluated once and reused. The physical projection-pushdown rule merged the outer projection into the file `DataSourceExec`, inlining the aliased volatile expression at every reference site and turning one call into N independent calls, so the references diverged. This regressed in 52.0.0 and reproduces on file scans (parquet/csv) but not in-memory tables. Guard `FileScanConfig::try_swapping_with_projection`: decline the merge when a volatile expression in the scan's existing projection is referenced by the incoming projection, reusing the existing `is_volatile()` utility -- the same gate the physical `ProjectionPushdown` and `FilterPushdown` rules already apply. Deterministic expressions still merge freely. Closes apache#23220.
1 parent 7ff7278 commit c62b344

2 files changed

Lines changed: 238 additions & 1 deletion

File tree

‎datafusion/datasource/src/file_scan_config/mod.rs‎

Lines changed: 208 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,11 @@ use datafusion_expr::Operator;
4141
use crate::source::OpenArgs;
4242
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
4343
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
44+
use datafusion_physical_expr::utils::collect_columns;
4445
use datafusion_physical_expr::utils::reassign_expr_columns;
4546
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction};
4647
use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
47-
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
48+
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_volatile};
4849
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
4950
use datafusion_physical_plan::SortOrderPushdownResult;
5051
use datafusion_physical_plan::coop::cooperative;
@@ -649,6 +650,45 @@ fn project_output_partitioning(
649650
}
650651
}
651652

653+
/// Returns `true` if merging `outer` into `inner` would duplicate a volatile
654+
/// expression; the caller should then decline the merge.
655+
///
656+
/// `inner` is the scan's current projection and `outer` the projection being
657+
/// pushed into it; merging substitutes each `inner` expression into every
658+
/// `outer` reference to it. If a volatile `inner` expression (e.g. `random()`,
659+
/// `uuid()`) is referenced by `outer`, that one value gets inlined at each site
660+
/// and re-evaluated independently, so references meant to share a single
661+
/// "locked-in" value diverge. This is the volatility guard the physical
662+
/// `ProjectionPushdown` and `FilterPushdown` rules already apply (see
663+
/// `datafusion_physical_expr_common::physical_expr::is_volatile`).
664+
///
665+
/// A volatile `inner` expression blocks the merge whenever `outer` references
666+
/// it at all: `collect_columns` reports a column at most once per `outer`
667+
/// expression, so a self-duplicating expression such as `r + r` would slip past
668+
/// a stricter `> 1` threshold.
669+
fn would_duplicate_volatile_exprs(
670+
inner: &ProjectionExprs,
671+
outer: &ProjectionExprs,
672+
) -> bool {
673+
let inner_exprs = inner.as_ref();
674+
let outer_exprs = outer.as_ref();
675+
676+
let mut referenced = vec![false; inner_exprs.len()];
677+
for proj_expr in outer_exprs {
678+
for col in collect_columns(&proj_expr.expr) {
679+
let idx = col.index();
680+
if idx < referenced.len() {
681+
referenced[idx] = true;
682+
}
683+
}
684+
}
685+
686+
referenced
687+
.iter()
688+
.enumerate()
689+
.any(|(idx, &referenced)| referenced && is_volatile(&inner_exprs[idx].expr))
690+
}
691+
652692
impl DataSource for FileScanConfig {
653693
fn open(
654694
&self,
@@ -945,6 +985,15 @@ impl DataSource for FileScanConfig {
945985
&self,
946986
projection: &ProjectionExprs,
947987
) -> Result<Option<Arc<dyn DataSource>>> {
988+
// Don't merge a projection into the scan if it would inline a volatile
989+
// expression that the outer projection references, which would turn a
990+
// single "locked-in" value (e.g. `random()` aliased in a subquery) into
991+
// multiple independent evaluations. See #23220.
992+
if let Some(inner) = self.file_source.projection()
993+
&& would_duplicate_volatile_exprs(inner, projection)
994+
{
995+
return Ok(None);
996+
}
948997
match self.file_source.try_pushdown_projection(projection)? {
949998
Some(new_source) => {
950999
let mut new_file_scan_config = self.clone();
@@ -3342,4 +3391,162 @@ mod tests {
33423391
);
33433392
Ok(())
33443393
}
3394+
3395+
/// Helper: build a `ProjectionExprs` from `(expr, alias)` pairs.
3396+
fn make_projection(pairs: Vec<(Arc<dyn PhysicalExpr>, &str)>) -> ProjectionExprs {
3397+
ProjectionExprs::new(
3398+
pairs
3399+
.into_iter()
3400+
.map(|(expr, alias)| ProjectionExpr::new(expr, alias)),
3401+
)
3402+
}
3403+
3404+
/// Helper: create a volatile (non-deterministic) function expression,
3405+
/// e.g. `random()`.
3406+
fn make_volatile_expr() -> Arc<dyn PhysicalExpr> {
3407+
use datafusion_common::config::ConfigOptions;
3408+
use datafusion_expr::ScalarUDF;
3409+
use datafusion_functions::math::random::RandomFunc;
3410+
use datafusion_physical_expr::ScalarFunctionExpr;
3411+
3412+
Arc::new(ScalarFunctionExpr::new(
3413+
"random",
3414+
Arc::new(ScalarUDF::from(RandomFunc::new())),
3415+
vec![],
3416+
Arc::new(Field::new("random", DataType::Float64, false)),
3417+
Arc::new(ConfigOptions::default()),
3418+
))
3419+
}
3420+
3421+
/// Column-only inner projections always merge safely, even when
3422+
/// the outer projection references them multiple times.
3423+
#[test]
3424+
fn test_would_duplicate_allows_column_only_inner() {
3425+
let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
3426+
let col_b: Arc<dyn PhysicalExpr> = Arc::new(Column::new("b", 1));
3427+
3428+
let inner =
3429+
make_projection(vec![(Arc::clone(&col_a), "a"), (Arc::clone(&col_b), "b")]);
3430+
3431+
// Outer references col 0 twice
3432+
let outer = make_projection(vec![
3433+
(Arc::new(Column::new("a", 0)), "x"),
3434+
(Arc::new(Column::new("a", 0)), "y"),
3435+
]);
3436+
3437+
assert!(!would_duplicate_volatile_exprs(&inner, &outer));
3438+
}
3439+
3440+
/// Deterministic computed expressions (arithmetic) referenced multiple
3441+
/// times are allowed to merge — only volatile expressions are protected.
3442+
#[test]
3443+
fn test_would_duplicate_allows_deterministic_computed_multi_ref() {
3444+
let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
3445+
let col_b: Arc<dyn PhysicalExpr> = Arc::new(Column::new("b", 1));
3446+
// Inner: [a + b, b] (index 0 is deterministic computed)
3447+
let inner = make_projection(vec![
3448+
(
3449+
Arc::new(BinaryExpr::new(
3450+
Arc::clone(&col_a),
3451+
Operator::Plus,
3452+
Arc::clone(&col_b),
3453+
)),
3454+
"sum",
3455+
),
3456+
(Arc::clone(&col_b), "b"),
3457+
]);
3458+
3459+
// Outer references index 0 twice
3460+
let outer = make_projection(vec![
3461+
(Arc::new(Column::new("sum", 0)), "x"),
3462+
(Arc::new(Column::new("sum", 0)), "y"),
3463+
]);
3464+
3465+
// Deterministic arithmetic → allow merge even though duplicated
3466+
assert!(!would_duplicate_volatile_exprs(&inner, &outer));
3467+
}
3468+
3469+
/// A volatile expression the outer projection does not reference is
3470+
/// safe to merge (it is projected away, not duplicated).
3471+
#[test]
3472+
fn test_would_duplicate_allows_unreferenced_volatile() {
3473+
let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
3474+
// Inner: [random(), a]
3475+
let inner =
3476+
make_projection(vec![(make_volatile_expr(), "r"), (Arc::clone(&col_a), "a")]);
3477+
3478+
// Outer references only index 1 (the column), not the volatile expr
3479+
let outer = make_projection(vec![(Arc::new(Column::new("a", 1)), "a")]);
3480+
3481+
assert!(!would_duplicate_volatile_exprs(&inner, &outer));
3482+
}
3483+
3484+
/// A volatile expression referenced multiple times must block merge:
3485+
/// this is the #23220 regression (`random()` aliased then referenced as
3486+
/// `x` and `y`).
3487+
#[test]
3488+
fn test_would_duplicate_blocks_multi_ref_volatile() {
3489+
// Inner: [random()]
3490+
let inner = make_projection(vec![(make_volatile_expr(), "r")]);
3491+
3492+
// Outer references index 0 twice
3493+
let outer = make_projection(vec![
3494+
(Arc::new(Column::new("r", 0)), "x"),
3495+
(Arc::new(Column::new("r", 0)), "y"),
3496+
]);
3497+
3498+
assert!(would_duplicate_volatile_exprs(&inner, &outer));
3499+
}
3500+
3501+
/// A volatile expression referenced by a single outer expression must
3502+
/// also block merge, since `collect_columns` collapses repeated uses
3503+
/// within one expression (e.g. `r + r`) — the `>= 1` threshold.
3504+
#[test]
3505+
fn test_would_duplicate_blocks_single_expr_self_ref_volatile() {
3506+
// Inner: [random()]
3507+
let inner = make_projection(vec![(make_volatile_expr(), "r")]);
3508+
3509+
// Outer: [r + r] — one expression, but it duplicates `random()`
3510+
let outer = make_projection(vec![(
3511+
Arc::new(BinaryExpr::new(
3512+
Arc::new(Column::new("r", 0)),
3513+
Operator::Plus,
3514+
Arc::new(Column::new("r", 0)),
3515+
)),
3516+
"x",
3517+
)]);
3518+
3519+
assert!(would_duplicate_volatile_exprs(&inner, &outer));
3520+
}
3521+
3522+
/// A volatile expression buried inside a larger expression (e.g.
3523+
/// `random() + 1`) is still detected and blocks merge.
3524+
#[test]
3525+
fn test_would_duplicate_blocks_volatile_nested_in_arithmetic() {
3526+
// Inner: [random() + 1]
3527+
let inner = make_projection(vec![(
3528+
Arc::new(BinaryExpr::new(
3529+
make_volatile_expr(),
3530+
Operator::Plus,
3531+
Arc::new(Literal::new(ScalarValue::Float64(Some(1.0)))),
3532+
)),
3533+
"expr",
3534+
)]);
3535+
3536+
// Outer references index 0 twice
3537+
let outer = make_projection(vec![
3538+
(Arc::new(Column::new("expr", 0)), "x"),
3539+
(Arc::new(Column::new("expr", 0)), "y"),
3540+
]);
3541+
3542+
assert!(would_duplicate_volatile_exprs(&inner, &outer));
3543+
}
3544+
3545+
/// Empty projections should not block merging.
3546+
#[test]
3547+
fn test_would_duplicate_empty_projections() {
3548+
let inner = make_projection(vec![]);
3549+
let outer = make_projection(vec![]);
3550+
assert!(!would_duplicate_volatile_exprs(&inner, &outer));
3551+
}
33453552
}

‎datafusion/sqllogictest/test_files/projection_pushdown.slt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2074,3 +2074,33 @@ SELECT s, id FROM simple_struct WHERE s['value'] > 100 AND id < 4;
20742074
# reset it explicitly.
20752075
statement ok
20762076
SET datafusion.execution.target_partitions = 4;
2077+
2078+
#####################
2079+
# Section: volatile expressions are not duplicated by projection pushdown
2080+
#
2081+
# Regression test for #23220: a volatile expression (e.g. `random()`) aliased
2082+
# once in a subquery and referenced multiple times must be evaluated once and
2083+
# reused. Projection pushdown must not merge the outer projection into the file
2084+
# scan when doing so would inline and duplicate the volatile expression.
2085+
# Reproduces only against a file scan (not an in-memory table); if the volatile
2086+
# expression is duplicated, the two references diverge and `x = y` is false.
2087+
#####################
2088+
2089+
statement ok
2090+
COPY (SELECT 1 AS id UNION ALL SELECT 2 UNION ALL SELECT 3)
2091+
TO 'test_files/scratch/projection_pushdown/volatile.parquet'
2092+
STORED AS PARQUET;
2093+
2094+
statement ok
2095+
CREATE EXTERNAL TABLE volatile_scan STORED AS PARQUET
2096+
LOCATION 'test_files/scratch/projection_pushdown/volatile.parquet';
2097+
2098+
# The two references to the aliased `random()` value must be equal on every
2099+
# row: the expression is evaluated once and reused, not inlined twice.
2100+
query B rowsort
2101+
SELECT s.x = s.y
2102+
FROM (SELECT r AS x, r AS y FROM (SELECT random() AS r FROM volatile_scan) AS t) AS s;
2103+
----
2104+
true
2105+
true
2106+
true

0 commit comments

Comments
 (0)