Skip to content

Commit e408875

Browse files
adriangbclaude
andcommitted
feat: make the materialized CTE nodes first-class LogicalPlan variants
Replace the `MaterializedCte` and `MaterializedCteScan` extension nodes with the variants `LogicalPlan::MaterializedCte` and `LogicalPlan::MaterializedCteScan`. Each rule now states what it does with the nodes in its `match`, and the downcasts are removed from `PushDownFilter`, `PullUpCorrelatedExpr`, `head_output_expr` and the physical planner. `MaterializedCte::try_new` checks that the scans have the schema of the body, and `MaterializedCteScan::find_in` finds the scans of one CTE, also in subquery expressions. The unparser, the Substrait producer and the protobuf serializer return a "not implemented" error for the two variants. Plans and results do not change. Closes apache#26238. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5009f42 commit e408875

18 files changed

Lines changed: 521 additions & 328 deletions

File tree

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 40 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,9 @@ use datafusion_expr::physical_planning_context::{
9494
use datafusion_expr::utils::{expr_to_columns, split_conjunction};
9595
use datafusion_expr::{
9696
Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,
97-
FetchType, Filter, JoinType, Operator, RecursiveQuery, SkipType, StringifiedPlan,
98-
WindowFrame, WindowFrameBound, WriteOp,
97+
FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCteScan,
98+
Operator, RecursiveQuery, SkipType, StringifiedPlan, WindowFrame, WindowFrameBound,
99+
WriteOp,
99100
};
100101
use datafusion_physical_expr::aggregate::{
101102
AggregateFunctionExpr, LoweredAggregate, LoweredAggregateBuilder,
@@ -260,19 +261,11 @@ impl DefaultPhysicalPlanner {
260261
///
261262
/// Returns `None` when `plan` has no materialized CTE.
262263
///
263-
/// [`MaterializedCte`]: datafusion_expr::MaterializedCte
264264
/// [`ViewTable`]: crate::datasource::ViewTable
265265
fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>> {
266-
fn as_cte(plan: &LogicalPlan) -> Option<&datafusion_expr::MaterializedCte> {
267-
match plan {
268-
LogicalPlan::Extension(Extension { node }) => node.as_any().downcast_ref(),
269-
_ => None,
270-
}
271-
}
272-
273266
let mut has_cte = false;
274267
plan.apply_with_subqueries(|node| {
275-
if as_cte(node).is_some() {
268+
if matches!(node, LogicalPlan::MaterializedCte(_)) {
276269
has_cte = true;
277270
return Ok(TreeNodeRecursion::Stop);
278271
}
@@ -284,53 +277,33 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
284277

285278
plan.clone()
286279
.transform_down_with_subqueries(|node| {
287-
let Some(cte) = as_cte(&node) else {
280+
let LogicalPlan::MaterializedCte(cte) = node else {
288281
return Ok(Transformed::no(node));
289282
};
290283
let old_id = cte.id;
291-
let new_id = datafusion_expr::MaterializedCteId::next();
292-
let continuation = cte
293-
.continuation
294-
.clone()
295-
.transform_down_with_subqueries(|node| {
284+
let new_id = MaterializedCteId::next();
285+
let continuation = Arc::unwrap_or_clone(cte.continuation)
286+
.transform_down_with_subqueries(|node| match node {
296287
// A nested occurrence of the same CTE owns the scans below
297288
// it, and is renumbered on its own.
298-
if as_cte(&node).is_some_and(|nested| nested.id == old_id) {
299-
return Ok(Transformed::new(
300-
node,
301-
false,
302-
TreeNodeRecursion::Jump,
303-
));
289+
LogicalPlan::MaterializedCte(ref nested) if nested.id == old_id => {
290+
Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
304291
}
305-
let LogicalPlan::Extension(Extension { node: scan }) = &node else {
306-
return Ok(Transformed::no(node));
307-
};
308-
match scan
309-
.as_any()
310-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
311-
{
312-
Some(scan) if scan.id == old_id => {
313-
let scan = datafusion_expr::MaterializedCteScan {
314-
id: new_id,
315-
..scan.clone()
316-
};
317-
Ok(Transformed::yes(LogicalPlan::Extension(Extension {
318-
node: Arc::new(scan),
319-
})))
320-
}
321-
_ => Ok(Transformed::no(node)),
292+
LogicalPlan::MaterializedCteScan(scan) if scan.id == old_id => {
293+
Ok(Transformed::yes(LogicalPlan::MaterializedCteScan(
294+
MaterializedCteScan { id: new_id, ..scan },
295+
)))
322296
}
297+
_ => Ok(Transformed::no(node)),
323298
})?
324299
.data;
325-
let cte = datafusion_expr::MaterializedCte {
326-
id: new_id,
327-
name: cte.name.clone(),
328-
cte: cte.cte.clone(),
329-
continuation,
330-
};
331-
Ok(Transformed::yes(LogicalPlan::Extension(Extension {
332-
node: Arc::new(cte),
333-
})))
300+
Ok(Transformed::yes(LogicalPlan::MaterializedCte(
301+
MaterializedCte {
302+
id: new_id,
303+
continuation: Arc::new(continuation),
304+
..cte
305+
},
306+
)))
334307
})
335308
.map(|t| Some(t.data))
336309
}
@@ -838,6 +811,16 @@ impl DefaultPhysicalPlanner {
838811
self.plan_describe(&Arc::clone(schema), output_schema)?
839812
}
840813

814+
LogicalPlan::MaterializedCteScan(scan) => {
815+
// Unbound until `bind_materialized_cte_scans` runs on the whole plan.
816+
Arc::new(MaterializedCteScanExec::new(
817+
scan.id.as_u64(),
818+
&scan.name,
819+
Arc::clone(scan.schema.inner()),
820+
session_state.config().target_partitions(),
821+
))
822+
}
823+
841824
// 1 Child
842825
LogicalPlan::Copy(CopyTo {
843826
input,
@@ -2051,31 +2034,10 @@ impl DefaultPhysicalPlanner {
20512034
)?)
20522035
}
20532036

2054-
// N Children
2055-
LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
2056-
LogicalPlan::Extension(Extension { node })
2057-
if node.as_any().is::<datafusion_expr::MaterializedCte>() =>
2058-
{
2059-
let cte = node
2060-
.as_any()
2061-
.downcast_ref::<datafusion_expr::MaterializedCte>()
2062-
.unwrap();
2037+
LogicalPlan::MaterializedCte(cte) => {
20632038
let [body, continuation] = children.two()?;
20642039
// A CTE that nothing reads is never evaluated.
2065-
let mut is_referenced = false;
2066-
cte.continuation.apply_with_subqueries(|node| {
2067-
if let LogicalPlan::Extension(Extension { node }) = node
2068-
&& let Some(scan) =
2069-
node.as_any()
2070-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
2071-
&& scan.id == cte.id
2072-
{
2073-
is_referenced = true;
2074-
return Ok(TreeNodeRecursion::Stop);
2075-
}
2076-
Ok(TreeNodeRecursion::Continue)
2077-
})?;
2078-
if !is_referenced {
2040+
if cte.scans()?.is_empty() {
20792041
return Ok(continuation);
20802042
}
20812043
// The body is fully buffered before a scan yields a row, so an
@@ -2090,21 +2052,9 @@ impl DefaultPhysicalPlanner {
20902052
Arc::new(MaterializedCteBuffer::new(cte.id.as_u64(), &cte.name));
20912053
Arc::new(MaterializedCteExec::new(body, continuation, buffer))
20922054
}
2093-
LogicalPlan::Extension(Extension { node })
2094-
if node.as_any().is::<datafusion_expr::MaterializedCteScan>() =>
2095-
{
2096-
let scan = node
2097-
.as_any()
2098-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
2099-
.unwrap();
2100-
// Unbound until `bind_materialized_cte_scans` runs on the whole plan.
2101-
Arc::new(MaterializedCteScanExec::new(
2102-
scan.id.as_u64(),
2103-
&scan.name,
2104-
Arc::clone(scan.schema.inner()),
2105-
session_state.config().target_partitions(),
2106-
))
2107-
}
2055+
2056+
// N Children
2057+
LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
21082058
LogicalPlan::Extension(Extension { node }) => {
21092059
let mut maybe_plan = None;
21102060
let children = children.vec();
@@ -2710,7 +2660,9 @@ fn extract_dml_filters(
27102660
| LogicalPlan::Ddl(_)
27112661
| LogicalPlan::Copy(_)
27122662
| LogicalPlan::Unnest(_)
2713-
| LogicalPlan::RecursiveQuery(_) => {
2663+
| LogicalPlan::RecursiveQuery(_)
2664+
| LogicalPlan::MaterializedCte(_)
2665+
| LogicalPlan::MaterializedCteScan(_) => {
27142666
// No filters to extract from leaf/meta plans
27152667
}
27162668
// Plans with inputs (may contain filters in children)

‎datafusion/expr/src/logical_plan/display.rs‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ use std::fmt;
2222

2323
use crate::{
2424
Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,
25-
Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition,
26-
Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest,
27-
Values, Window, expr_vec_fmt,
25+
Join, Limit, LogicalPlan, MaterializedCte, MaterializedCteScan, Partitioning,
26+
Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias,
27+
TableProviderFilterPushDown, TableScan, Unnest, Values, Window, expr_vec_fmt,
2828
};
2929

3030
use crate::dml::CopyTo;
@@ -691,6 +691,18 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {
691691
"Detail": format!("{:?}", e.node)
692692
)
693693
}
694+
LogicalPlan::MaterializedCte(MaterializedCte { name, .. }) => {
695+
pg_fields!(
696+
"Node Type": "MaterializedCte",
697+
"Name": name,
698+
)
699+
}
700+
LogicalPlan::MaterializedCteScan(MaterializedCteScan { name, .. }) => {
701+
pg_fields!(
702+
"Node Type": "MaterializedCteScan",
703+
"Name": name,
704+
)
705+
}
694706
LogicalPlan::DescribeTable(DescribeTable { .. }) => {
695707
pg_fields!(
696708
"Node Type": "DescribeTable"

‎datafusion/expr/src/logical_plan/invariants.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,9 @@ fn check_inner_plan(inner_plan: &LogicalPlan) -> Result<()> {
355355
}
356356
},
357357
LogicalPlan::Extension(_) => Ok(()),
358+
// A materialized CTE is not checked, as when it was an extension
359+
// node. `PullUpCorrelatedExpr` rewrites its continuation, or stops.
360+
LogicalPlan::MaterializedCte(_) => Ok(()),
358361
plan => check_no_outer_references(plan),
359362
}
360363
}

0 commit comments

Comments
 (0)