Skip to content

Commit 1724943

Browse files
adriangbclaude
andcommitted
feat: protobuf serialization for the materialized CTE logical plan nodes
Encode and decode LogicalPlan::MaterializedCte and LogicalPlan::MaterializedCteScan, replacing the not-implemented errors. The ids of the encoded plan are kept on decode (via the new MaterializedCteId::from_u64), so the scans in the continuation, which are decoded before their CTE node, stay bound to it. Reusing ids from another process is safe in a logical plan: a scan reads the nearest MaterializedCte above it that has its id, and the physical planner gives every CTE a new id. MaterializedCte::try_new checks the scan schemas. Round-trip tests cover the plan before and after the optimizer, check that the decoded plan is equal to the original, and that it returns the same rows. Together with the previous commits, this adds protobuf encodings for LogicalPlan::MaterializedCte, LogicalPlan::MaterializedCteScan, MaterializedCteExec and MaterializedCteScanExec. A decoded physical plan fragment that has a scan without its MaterializedCteExec fails with a clear error when the scan is executed. Closes apache#26231. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 07f44de commit 1724943

3 files changed

Lines changed: 141 additions & 11 deletions

File tree

‎datafusion/expr/src/logical_plan/materialized_cte.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ impl MaterializedCteId {
5757
pub fn as_u64(self) -> u64 {
5858
self.0
5959
}
60+
61+
/// Create an id from the value that [`Self::as_u64`] returned, to decode
62+
/// a serialized plan.
63+
///
64+
/// Such an id comes from another process, so it can be equal to an id
65+
/// that [`Self::next`] returns in this process. This is safe in a logical
66+
/// plan: a scan reads the nearest [`MaterializedCte`] above it that has
67+
/// its id, and the physical planner gives every CTE a new id.
68+
pub fn from_u64(id: u64) -> Self {
69+
Self(id)
70+
}
6071
}
6172

6273
impl fmt::Display for MaterializedCteId {

‎datafusion/proto/src/logical_plan/mod.rs‎

Lines changed: 67 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use datafusion_common::format::{
4141
};
4242
use datafusion_common::utils::{usize_from_wire, usize_to_wire};
4343
use datafusion_common::{
44-
NullEquality, Result, TableReference, assert_or_internal_err, context,
44+
DFSchema, NullEquality, Result, TableReference, assert_or_internal_err, context,
4545
internal_datafusion_err, internal_err, not_impl_err, plan_err,
4646
};
4747
use datafusion_datasource::file_format::FileFormat;
@@ -59,8 +59,9 @@ use datafusion_datasource_json::file_format::{
5959
use datafusion_datasource_parquet::file_format::{ParquetFormat, ParquetFormatFactory};
6060
use datafusion_expr::dml::InsertOp;
6161
use datafusion_expr::{
62-
AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, Operator, RangePartitioning,
63-
RecursiveQuery, SkipType, TableSource, Unnest, WriteOp,
62+
AggregateUDF, DmlStatement, FetchType, HigherOrderUDF, MaterializedCte,
63+
MaterializedCteId, MaterializedCtePlan, MaterializedCteScan, Operator,
64+
RangePartitioning, RecursiveQuery, SkipType, TableSource, Unnest, WriteOp,
6465
};
6566
use datafusion_expr::{
6667
DistinctOn, DropView, Expr, JoinConstraint, LogicalPlan, LogicalPlanBuilder,
@@ -1355,11 +1356,42 @@ impl AsLogicalPlan for LogicalPlanNode {
13551356
)
13561357
.map(LogicalPlan::RecursiveQuery)
13571358
}
1358-
LogicalPlanType::MaterializedCte(_)
1359-
| LogicalPlanType::MaterializedCteScan(_) => {
1360-
not_impl_err!(
1361-
"LogicalPlan serde is not yet implemented for MATERIALIZED CTEs"
1359+
LogicalPlanType::MaterializedCte(cte_node) => {
1360+
let body = cte_node
1361+
.cte
1362+
.as_ref()
1363+
.ok_or_else(|| internal_datafusion_err!(
1364+
"Protobuf deserialization error, MaterializedCteNode was missing required field cte."
1365+
))?
1366+
.try_into_logical_plan(ctx, extension_codec)?;
1367+
let continuation = cte_node
1368+
.continuation
1369+
.as_ref()
1370+
.ok_or_else(|| internal_datafusion_err!(
1371+
"Protobuf deserialization error, MaterializedCteNode was missing required field continuation."
1372+
))?
1373+
.try_into_logical_plan(ctx, extension_codec)?;
1374+
1375+
// The ids of the encoded plan are kept, so the scans in the
1376+
// continuation, which are decoded before this node, stay
1377+
// bound to it. `try_new` checks their schemas.
1378+
MaterializedCte::try_new(
1379+
MaterializedCteId::from_u64(cte_node.id),
1380+
cte_node.name.clone(),
1381+
Arc::new(body),
1382+
Arc::new(continuation),
13621383
)
1384+
.map(|cte| LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)))
1385+
}
1386+
LogicalPlanType::MaterializedCteScan(scan) => {
1387+
let schema: DFSchema = convert_required!(scan.schema)?;
1388+
Ok(LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(
1389+
MaterializedCteScan::new(
1390+
MaterializedCteId::from_u64(scan.id),
1391+
scan.name.clone(),
1392+
Arc::new(schema),
1393+
),
1394+
)))
13631395
}
13641396
LogicalPlanType::CteWorkTableScan(cte_work_table_scan_node) => {
13651397
let CteWorkTableScanNode { name, schema } = cte_work_table_scan_node;
@@ -2417,11 +2449,35 @@ impl AsLogicalPlan for LogicalPlanNode {
24172449
LogicalPlan::DescribeTable(_) => Err(proto_error(
24182450
"LogicalPlan serde is not yet implemented for DescribeTable",
24192451
)),
2420-
LogicalPlan::MaterializedCte(_) => {
2421-
not_impl_err!(
2422-
"LogicalPlan serde is not yet implemented for MATERIALIZED CTEs"
2423-
)
2452+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
2453+
let body = LogicalPlanNode::try_from_logical_plan(
2454+
cte.body().as_ref(),
2455+
extension_codec,
2456+
)?;
2457+
let continuation = LogicalPlanNode::try_from_logical_plan(
2458+
cte.continuation().as_ref(),
2459+
extension_codec,
2460+
)?;
2461+
Ok(LogicalPlanNode {
2462+
logical_plan_type: Some(LogicalPlanType::MaterializedCte(Box::new(
2463+
protobuf::MaterializedCteNode {
2464+
id: cte.id().as_u64(),
2465+
name: cte.name().to_string(),
2466+
cte: Some(Box::new(body)),
2467+
continuation: Some(Box::new(continuation)),
2468+
},
2469+
))),
2470+
})
24242471
}
2472+
LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => Ok(LogicalPlanNode {
2473+
logical_plan_type: Some(LogicalPlanType::MaterializedCteScan(
2474+
protobuf::MaterializedCteScanNode {
2475+
id: scan.id().as_u64(),
2476+
name: scan.name().to_string(),
2477+
schema: Some(scan.schema().try_into()?),
2478+
},
2479+
)),
2480+
}),
24252481
LogicalPlan::RecursiveQuery(recursive) => {
24262482
let static_term = LogicalPlanNode::try_from_logical_plan(
24272483
recursive.static_term.as_ref(),

‎datafusion/proto/tests/cases/roundtrip_logical_plan.rs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use datafusion_common::parquet_config::{
3636
DFParquetWriterVersion, RowGroupRangeAssignment,
3737
};
3838
use datafusion_common::parsers::CompressionTypeVariant;
39+
use datafusion_expr::logical_plan::MaterializedCtePlan;
3940
use datafusion_functions_aggregate::sum::sum_distinct;
4041
use prost::Message;
4142
use std::any::Any;
@@ -3602,6 +3603,68 @@ async fn roundtrip_recursive_query() {
36023603
);
36033604
}
36043605

3606+
/// Round trip the plan of `sql`, which uses a materialized CTE, before and
3607+
/// after the optimizer. The decoded plans must return the rows of `sql`.
3608+
async fn roundtrip_materialized_cte(sql: &str, scans: usize) -> Result<()> {
3609+
// sqlparser only parses `MATERIALIZED` for the PostgreSQL dialect.
3610+
let config =
3611+
SessionConfig::new().set_str("datafusion.sql_parser.dialect", "PostgreSQL");
3612+
let ctx = SessionContext::new_with_config(config.clone());
3613+
let dataframe = ctx.sql(sql).await?;
3614+
let plans = [
3615+
dataframe.logical_plan().clone(),
3616+
dataframe.clone().into_optimized_plan()?,
3617+
];
3618+
let output = dataframe.collect().await?;
3619+
3620+
for plan in plans {
3621+
let bytes = logical_plan_to_bytes(&plan)?;
3622+
let ctx = SessionContext::new_with_config(config.clone());
3623+
let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx.task_ctx())?;
3624+
// The ids are part of the debug format: the scans keep the id of
3625+
// their CTE.
3626+
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}"));
3627+
assert_eq!(plan, logical_round_trip);
3628+
3629+
let mut cte_scans = vec![];
3630+
logical_round_trip.apply_with_subqueries(|node| {
3631+
if let LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) = node {
3632+
cte_scans.push(cte.scans()?.len());
3633+
}
3634+
Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
3635+
})?;
3636+
assert_eq!(cte_scans, vec![scans]);
3637+
3638+
let dataframe = ctx.execute_logical_plan(logical_round_trip).await?;
3639+
let output_round_trip = dataframe.collect().await?;
3640+
assert_eq!(
3641+
format!("{}", pretty_format_batches(&output)?),
3642+
format!("{}", pretty_format_batches(&output_round_trip)?)
3643+
);
3644+
}
3645+
Ok(())
3646+
}
3647+
3648+
#[tokio::test]
3649+
async fn roundtrip_materialized_cte_with_two_scans() -> Result<()> {
3650+
roundtrip_materialized_cte(
3651+
"WITH c AS MATERIALIZED (SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(v))
3652+
SELECT a.v, b.v FROM c AS a JOIN c AS b ON a.v + 1 = b.v ORDER BY a.v",
3653+
2,
3654+
)
3655+
.await
3656+
}
3657+
3658+
#[tokio::test]
3659+
async fn roundtrip_materialized_cte_scan_in_scalar_subquery() -> Result<()> {
3660+
roundtrip_materialized_cte(
3661+
"WITH c AS MATERIALIZED (SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(v))
3662+
SELECT v FROM c WHERE v < (SELECT max(v) FROM c) ORDER BY v",
3663+
2,
3664+
)
3665+
.await
3666+
}
3667+
36053668
#[tokio::test]
36063669
async fn roundtrip_union_query() -> Result<()> {
36073670
let query = "SELECT a FROM t1

0 commit comments

Comments
 (0)