Skip to content

Commit 37c5b98

Browse files
adriangbclaude
andcommitted
feat: make the materialized CTE nodes a first-class LogicalPlan variant
Replace the `MaterializedCte` and `MaterializedCteScan` extension nodes with the variants `LogicalPlan::MaterializedCte` and `LogicalPlan::MaterializedCteScan`. Each rule now states what it does with the nodes in its `match`, and the downcasts are removed from `PushDownFilter`, `PullUpCorrelatedExpr`, `head_output_expr` and the physical planner. `MaterializedCte::try_new` checks that the scans have the schema of the body, and `MaterializedCteScan::find_in` finds the scans of one CTE, also in subquery expressions. The unparser, the Substrait producer and the protobuf serializer return a "not implemented" error for the two variants. Plans and results do not change. Closes apache#26238. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 5009f42 commit 37c5b98

19 files changed

Lines changed: 564 additions & 325 deletions

File tree

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 46 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,8 @@ use datafusion_expr::physical_planning_context::{
9494
use datafusion_expr::utils::{expr_to_columns, split_conjunction};
9595
use datafusion_expr::{
9696
Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,
97-
FetchType, Filter, JoinType, Operator, RecursiveQuery, SkipType, StringifiedPlan,
97+
FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan,
98+
MaterializedCteScan, Operator, RecursiveQuery, SkipType, StringifiedPlan,
9899
WindowFrame, WindowFrameBound, WriteOp,
99100
};
100101
use datafusion_physical_expr::aggregate::{
@@ -260,19 +261,14 @@ impl DefaultPhysicalPlanner {
260261
///
261262
/// Returns `None` when `plan` has no materialized CTE.
262263
///
263-
/// [`MaterializedCte`]: datafusion_expr::MaterializedCte
264264
/// [`ViewTable`]: crate::datasource::ViewTable
265265
fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>> {
266-
fn as_cte(plan: &LogicalPlan) -> Option<&datafusion_expr::MaterializedCte> {
267-
match plan {
268-
LogicalPlan::Extension(Extension { node }) => node.as_any().downcast_ref(),
269-
_ => None,
270-
}
271-
}
272-
273266
let mut has_cte = false;
274267
plan.apply_with_subqueries(|node| {
275-
if as_cte(node).is_some() {
268+
if matches!(
269+
node,
270+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_))
271+
) {
276272
has_cte = true;
277273
return Ok(TreeNodeRecursion::Stop);
278274
}
@@ -284,53 +280,40 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
284280

285281
plan.clone()
286282
.transform_down_with_subqueries(|node| {
287-
let Some(cte) = as_cte(&node) else {
283+
let LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) = node else {
288284
return Ok(Transformed::no(node));
289285
};
290286
let old_id = cte.id;
291-
let new_id = datafusion_expr::MaterializedCteId::next();
292-
let continuation = cte
293-
.continuation
294-
.clone()
295-
.transform_down_with_subqueries(|node| {
287+
let new_id = MaterializedCteId::next();
288+
let continuation = Arc::unwrap_or_clone(cte.continuation)
289+
.transform_down_with_subqueries(|node| match node {
296290
// A nested occurrence of the same CTE owns the scans below
297291
// it, and is renumbered on its own.
298-
if as_cte(&node).is_some_and(|nested| nested.id == old_id) {
299-
return Ok(Transformed::new(
300-
node,
301-
false,
302-
TreeNodeRecursion::Jump,
303-
));
292+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
293+
ref nested,
294+
)) if nested.id == old_id => {
295+
Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
304296
}
305-
let LogicalPlan::Extension(Extension { node: scan }) = &node else {
306-
return Ok(Transformed::no(node));
307-
};
308-
match scan
309-
.as_any()
310-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
297+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))
298+
if scan.id == old_id =>
311299
{
312-
Some(scan) if scan.id == old_id => {
313-
let scan = datafusion_expr::MaterializedCteScan {
300+
Ok(Transformed::yes(LogicalPlan::MaterializedCte(
301+
MaterializedCtePlan::Scan(MaterializedCteScan {
314302
id: new_id,
315-
..scan.clone()
316-
};
317-
Ok(Transformed::yes(LogicalPlan::Extension(Extension {
318-
node: Arc::new(scan),
319-
})))
320-
}
321-
_ => Ok(Transformed::no(node)),
303+
..scan
304+
}),
305+
)))
322306
}
307+
_ => Ok(Transformed::no(node)),
323308
})?
324309
.data;
325-
let cte = datafusion_expr::MaterializedCte {
326-
id: new_id,
327-
name: cte.name.clone(),
328-
cte: cte.cte.clone(),
329-
continuation,
330-
};
331-
Ok(Transformed::yes(LogicalPlan::Extension(Extension {
332-
node: Arc::new(cte),
333-
})))
310+
Ok(Transformed::yes(LogicalPlan::MaterializedCte(
311+
MaterializedCtePlan::Cte(MaterializedCte {
312+
id: new_id,
313+
continuation: Arc::new(continuation),
314+
..cte
315+
}),
316+
)))
334317
})
335318
.map(|t| Some(t.data))
336319
}
@@ -838,6 +821,16 @@ impl DefaultPhysicalPlanner {
838821
self.plan_describe(&Arc::clone(schema), output_schema)?
839822
}
840823

824+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => {
825+
// Unbound until `bind_materialized_cte_scans` runs on the whole plan.
826+
Arc::new(MaterializedCteScanExec::new(
827+
scan.id.as_u64(),
828+
&scan.name,
829+
Arc::clone(scan.schema.inner()),
830+
session_state.config().target_partitions(),
831+
))
832+
}
833+
841834
// 1 Child
842835
LogicalPlan::Copy(CopyTo {
843836
input,
@@ -2051,31 +2044,10 @@ impl DefaultPhysicalPlanner {
20512044
)?)
20522045
}
20532046

2054-
// N Children
2055-
LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
2056-
LogicalPlan::Extension(Extension { node })
2057-
if node.as_any().is::<datafusion_expr::MaterializedCte>() =>
2058-
{
2059-
let cte = node
2060-
.as_any()
2061-
.downcast_ref::<datafusion_expr::MaterializedCte>()
2062-
.unwrap();
2047+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
20632048
let [body, continuation] = children.two()?;
20642049
// A CTE that nothing reads is never evaluated.
2065-
let mut is_referenced = false;
2066-
cte.continuation.apply_with_subqueries(|node| {
2067-
if let LogicalPlan::Extension(Extension { node }) = node
2068-
&& let Some(scan) =
2069-
node.as_any()
2070-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
2071-
&& scan.id == cte.id
2072-
{
2073-
is_referenced = true;
2074-
return Ok(TreeNodeRecursion::Stop);
2075-
}
2076-
Ok(TreeNodeRecursion::Continue)
2077-
})?;
2078-
if !is_referenced {
2050+
if cte.scans()?.is_empty() {
20792051
return Ok(continuation);
20802052
}
20812053
// The body is fully buffered before a scan yields a row, so an
@@ -2090,21 +2062,9 @@ impl DefaultPhysicalPlanner {
20902062
Arc::new(MaterializedCteBuffer::new(cte.id.as_u64(), &cte.name));
20912063
Arc::new(MaterializedCteExec::new(body, continuation, buffer))
20922064
}
2093-
LogicalPlan::Extension(Extension { node })
2094-
if node.as_any().is::<datafusion_expr::MaterializedCteScan>() =>
2095-
{
2096-
let scan = node
2097-
.as_any()
2098-
.downcast_ref::<datafusion_expr::MaterializedCteScan>()
2099-
.unwrap();
2100-
// Unbound until `bind_materialized_cte_scans` runs on the whole plan.
2101-
Arc::new(MaterializedCteScanExec::new(
2102-
scan.id.as_u64(),
2103-
&scan.name,
2104-
Arc::clone(scan.schema.inner()),
2105-
session_state.config().target_partitions(),
2106-
))
2107-
}
2065+
2066+
// N Children
2067+
LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
21082068
LogicalPlan::Extension(Extension { node }) => {
21092069
let mut maybe_plan = None;
21102070
let children = children.vec();
@@ -2710,7 +2670,8 @@ fn extract_dml_filters(
27102670
| LogicalPlan::Ddl(_)
27112671
| LogicalPlan::Copy(_)
27122672
| LogicalPlan::Unnest(_)
2713-
| LogicalPlan::RecursiveQuery(_) => {
2673+
| LogicalPlan::RecursiveQuery(_)
2674+
| LogicalPlan::MaterializedCte(_) => {
27142675
// No filters to extract from leaf/meta plans
27152676
}
27162677
// Plans with inputs (may contain filters in children)

‎datafusion/expr/src/logical_plan/display.rs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ use std::fmt;
2222

2323
use crate::{
2424
Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,
25-
Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition,
26-
Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest,
27-
Values, Window, expr_vec_fmt,
25+
Join, Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan,
26+
Partitioning, Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias,
27+
TableProviderFilterPushDown, TableScan, Unnest, Values, Window, expr_vec_fmt,
2828
};
2929

3030
use crate::dml::CopyTo;
@@ -691,6 +691,23 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {
691691
"Detail": format!("{:?}", e.node)
692692
)
693693
}
694+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
695+
name,
696+
..
697+
})) => {
698+
pg_fields!(
699+
"Node Type": "MaterializedCte",
700+
"Name": name,
701+
)
702+
}
703+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(
704+
MaterializedCteScan { name, .. },
705+
)) => {
706+
pg_fields!(
707+
"Node Type": "MaterializedCteScan",
708+
"Name": name,
709+
)
710+
}
694711
LogicalPlan::DescribeTable(DescribeTable { .. }) => {
695712
pg_fields!(
696713
"Node Type": "DescribeTable"

‎datafusion/expr/src/logical_plan/invariants.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use crate::logical_plan::MaterializedCtePlan;
1819
use datafusion_common::{
1920
DFSchemaRef, Result, assert_or_internal_err, plan_err,
2021
tree_node::{TreeNode, TreeNodeRecursion},
@@ -355,6 +356,9 @@ fn check_inner_plan(inner_plan: &LogicalPlan) -> Result<()> {
355356
}
356357
},
357358
LogicalPlan::Extension(_) => Ok(()),
359+
// A materialized CTE is not checked, as when it was an extension
360+
// node. `PullUpCorrelatedExpr` rewrites its continuation, or stops.
361+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_)) => Ok(()),
358362
plan => check_no_outer_references(plan),
359363
}
360364
}

0 commit comments

Comments
 (0)