Skip to content

Commit d03d393

Browse files
adriangbclaude
andcommitted
feat: protobuf serialization for the materialized CTE plan nodes
Add protobuf encodings for LogicalPlan::MaterializedCte, LogicalPlan::MaterializedCteScan, MaterializedCteExec and MaterializedCteScanExec. The physical encoding carries only the id of the CTE. The decode context holds one buffer for each id, so a decoded scan and its MaterializedCteExec share the buffer wherever they are in the plan, also when the scan is in a scalar subquery. Decoded buffers get new ids, because the ids of another process can be equal to local ids. A decoded plan fragment that has a scan without its MaterializedCteExec fails with a clear error when the scan is executed. Closes apache#26231. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 985bbbf commit d03d393

12 files changed

Lines changed: 1300 additions & 11 deletions

File tree

‎datafusion/expr/src/logical_plan/materialized_cte.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ impl MaterializedCteId {
5757
pub fn as_u64(self) -> u64 {
5858
self.0
5959
}
60+
61+
/// Create an id from the value that [`Self::as_u64`] returned, to decode
62+
/// a serialized plan.
63+
///
64+
/// Such an id comes from another process, so it can be equal to an id
65+
/// that [`Self::next`] returns in this process. This is safe in a logical
66+
/// plan: a scan reads the nearest [`MaterializedCte`] above it that has
67+
/// its id, and the physical planner gives every CTE a new id.
68+
pub fn from_u64(id: u64) -> Self {
69+
Self(id)
70+
}
6071
}
6172

6273
impl fmt::Display for MaterializedCteId {

‎datafusion/physical-plan/src/materialized_cte.rs‎

Lines changed: 203 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
//! the statistics of the body. The planner copies them to the scan with
4545
//! [`MaterializedCteScanExec::with_body_statistics`].
4646
47+
use std::collections::HashMap;
4748
use std::fmt;
4849
use std::pin::Pin;
4950
use std::sync::Arc;
@@ -53,10 +54,11 @@ use arrow::datatypes::SchemaRef;
5354
use arrow::record_batch::RecordBatch;
5455
use datafusion_common::config::ConfigOptions;
5556
use datafusion_common::tree_node::TreeNodeRecursion;
56-
use datafusion_common::{Result, Statistics, internal_err};
57+
use datafusion_common::{Result, Statistics, exec_err, internal_err};
5758
use datafusion_common_runtime::JoinSet;
5859
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
5960
use datafusion_execution::{SendableRecordBatchStream, SpillFile, TaskContext};
61+
use datafusion_expr::MaterializedCteId;
6062
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
6163
use futures::stream::BoxStream;
6264
use futures::{Stream, StreamExt, TryStreamExt};
@@ -304,6 +306,11 @@ impl MaterializedCteBuffer {
304306
self.id
305307
}
306308

309+
/// The name of the CTE.
310+
pub fn name(&self) -> &str {
311+
&self.name
312+
}
313+
307314
fn set_body(&self, body: Arc<dyn ExecutionPlan>) {
308315
*self.body.lock() = Some(body);
309316
}
@@ -318,8 +325,13 @@ impl MaterializedCteBuffer {
318325
self: &Arc<Self>,
319326
context: Arc<TaskContext>,
320327
) -> Result<OnceFut<MaterializedOutput>> {
328+
// A plan from the planner always has the body. A decoded plan does
329+
// not have it when the plan was split between the body and a scan.
321330
let Some(body) = self.body.lock().clone() else {
322-
return internal_err!("MaterializedCte {} has no body", self.name);
331+
return exec_err!(
332+
"The materialized CTE {} has a MaterializedCteScanExec but no MaterializedCteExec in this plan. The body and the scans of a materialized CTE must be in one plan and run in one process",
333+
self.name
334+
);
323335
};
324336
let once = Arc::clone(&self.output.lock());
325337
let name = self.name.clone();
@@ -331,6 +343,36 @@ impl MaterializedCteBuffer {
331343
}
332344
}
333345

346+
/// The buffers of the materialized CTEs in one decoded plan.
347+
///
348+
/// A [`MaterializedCteExec`] and its [`MaterializedCteScanExec`]s are encoded
349+
/// as separate nodes that have the same id. The decoder of each node asks for
350+
/// the buffer of that id, so all of them share one buffer, in any decode
351+
/// order. A scan in a scalar subquery is not below its `MaterializedCteExec`,
352+
/// so one registry must be used for the whole plan.
353+
///
354+
/// The ids in an encoded plan come from another process and can be equal to
355+
/// the ids of plans made in this process. Each buffer therefore gets a new id
356+
/// from [`MaterializedCteId::next`].
357+
#[derive(Debug, Default)]
358+
pub struct DecodedMaterializedCteBuffers {
359+
/// The buffers, by the id in the encoded plan.
360+
buffers: Mutex<HashMap<u64, Arc<MaterializedCteBuffer>>>,
361+
}
362+
363+
impl DecodedMaterializedCteBuffers {
364+
/// Return the buffer of the CTE that has the id `encoded_id` in the
365+
/// encoded plan. The first call for an id creates the buffer.
366+
pub fn buffer(&self, encoded_id: u64, name: &str) -> Arc<MaterializedCteBuffer> {
367+
let mut buffers = self.buffers.lock();
368+
let buffer = buffers.entry(encoded_id).or_insert_with(|| {
369+
let id = MaterializedCteId::next().as_u64();
370+
Arc::new(MaterializedCteBuffer::new(id, name))
371+
});
372+
Arc::clone(buffer)
373+
}
374+
}
375+
334376
async fn buffer_body(
335377
name: String,
336378
body: Arc<dyn ExecutionPlan>,
@@ -664,6 +706,81 @@ impl ExecutionPlan for MaterializedCteExec {
664706
// continuation handles it.
665707
Ok(FilterPushdownPropagation::if_any(child_pushdown_result))
666708
}
709+
710+
#[cfg(feature = "proto")]
711+
fn try_to_proto(
712+
&self,
713+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
714+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
715+
use datafusion_proto_models::protobuf;
716+
717+
// Destructure exhaustively (no `..`) so that adding a field to
718+
// `MaterializedCteExec` is a compile error here until it is either
719+
// serialized or explicitly documented as not needing to be.
720+
let Self {
721+
body,
722+
continuation,
723+
// Only the id and the name are encoded. The decoder makes a new
724+
// buffer and shares it with the scans that have the same id.
725+
buffer,
726+
// Copied from the continuation's properties, recomputed on decode.
727+
cache: _,
728+
} = self;
729+
Ok(Some(protobuf::PhysicalPlanNode {
730+
physical_plan_type: Some(
731+
protobuf::physical_plan_node::PhysicalPlanType::MaterializedCte(
732+
Box::new(protobuf::MaterializedCteExecNode {
733+
id: buffer.id(),
734+
name: buffer.name().to_string(),
735+
body: Some(Box::new(ctx.encode_child(body)?)),
736+
continuation: Some(Box::new(ctx.encode_child(continuation)?)),
737+
}),
738+
),
739+
),
740+
}))
741+
}
742+
}
743+
744+
#[cfg(feature = "proto")]
745+
impl MaterializedCteExec {
746+
/// Reconstruct a [`MaterializedCteExec`] from its protobuf representation.
747+
///
748+
/// The node shares the buffer that the decode context holds for its id
749+
/// with the decoded [`MaterializedCteScanExec`]s of the same id.
750+
pub fn try_from_proto(
751+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
752+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
753+
) -> Result<Arc<dyn ExecutionPlan>> {
754+
use datafusion_proto_models::protobuf;
755+
756+
let cte = crate::expect_plan_variant!(
757+
node,
758+
protobuf::physical_plan_node::PhysicalPlanType::MaterializedCte,
759+
"MaterializedCteExec",
760+
);
761+
// Destructure exhaustively so that a new field on
762+
// `MaterializedCteExecNode` is a compile error here rather than a
763+
// silently dropped field.
764+
let protobuf::MaterializedCteExecNode {
765+
id,
766+
name,
767+
body,
768+
continuation,
769+
} = &**cte;
770+
let body =
771+
ctx.decode_required_child(body.as_deref(), "MaterializedCteExec", "body")?;
772+
let continuation = ctx.decode_required_child(
773+
continuation.as_deref(),
774+
"MaterializedCteExec",
775+
"continuation",
776+
)?;
777+
let buffer = ctx.materialized_cte_buffer(*id, name)?;
778+
// A second node would replace the body that the scans read.
779+
if buffer.body.lock().is_some() {
780+
return internal_err!("Two MaterializedCteExec nodes have the id {id}");
781+
}
782+
Ok(Arc::new(Self::new(body, continuation, buffer)))
783+
}
667784
}
668785

669786
/// Reads the buffered output of a [`MaterializedCteExec`] body.
@@ -859,6 +976,90 @@ impl ExecutionPlan for MaterializedCteScanExec {
859976
_ => Ok(Arc::new(Statistics::new_unknown(&self.schema))),
860977
}
861978
}
979+
980+
#[cfg(feature = "proto")]
981+
fn try_to_proto(
982+
&self,
983+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
984+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
985+
use datafusion_common::utils::usize_to_wire;
986+
use datafusion_proto_models::protobuf;
987+
988+
// Destructure exhaustively (no `..`) so that adding a field to
989+
// `MaterializedCteScanExec` is a compile error here until it is
990+
// either serialized or explicitly documented as not needing to be.
991+
let Self {
992+
id,
993+
name,
994+
schema,
995+
// The decoder binds the scan to the buffer of `id` again.
996+
buffer: _,
997+
// Runtime metrics, created empty on decode.
998+
metrics: _,
999+
cache,
1000+
} = self;
1001+
let partitions = usize_to_wire(
1002+
cache.partitioning.partition_count(),
1003+
"MaterializedCteScanExec",
1004+
"partitions",
1005+
)?;
1006+
Ok(Some(protobuf::PhysicalPlanNode {
1007+
physical_plan_type: Some(
1008+
protobuf::physical_plan_node::PhysicalPlanType::MaterializedCteScan(
1009+
protobuf::MaterializedCteScanExecNode {
1010+
id: *id,
1011+
name: name.clone(),
1012+
schema: Some(schema.as_ref().try_into()?),
1013+
partitions,
1014+
},
1015+
),
1016+
),
1017+
}))
1018+
}
1019+
}
1020+
1021+
#[cfg(feature = "proto")]
1022+
impl MaterializedCteScanExec {
1023+
/// Reconstruct a [`MaterializedCteScanExec`] from its protobuf
1024+
/// representation.
1025+
///
1026+
/// The scan is bound to the buffer that the decode context holds for its
1027+
/// id. The decoded [`MaterializedCteExec`] of the same id gives that
1028+
/// buffer its body. If the decoded plan has no such node, the scan fails
1029+
/// when it is executed.
1030+
pub fn try_from_proto(
1031+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
1032+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
1033+
) -> Result<Arc<dyn ExecutionPlan>> {
1034+
use datafusion_common::utils::usize_from_wire;
1035+
use datafusion_proto_models::protobuf;
1036+
1037+
let scan = crate::expect_plan_variant!(
1038+
node,
1039+
protobuf::physical_plan_node::PhysicalPlanType::MaterializedCteScan,
1040+
"MaterializedCteScanExec",
1041+
);
1042+
// Destructure exhaustively so that a new field on
1043+
// `MaterializedCteScanExecNode` is a compile error here rather than a
1044+
// silently dropped field.
1045+
let protobuf::MaterializedCteScanExecNode {
1046+
id,
1047+
name,
1048+
schema,
1049+
partitions,
1050+
} = scan;
1051+
let schema = schema.as_ref().ok_or_else(|| {
1052+
datafusion_common::internal_datafusion_err!(
1053+
"MaterializedCteScanExec is missing required field 'schema'"
1054+
)
1055+
})?;
1056+
let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?);
1057+
let partitions =
1058+
usize_from_wire(*partitions, "MaterializedCteScanExec", "partitions")?;
1059+
let buffer = ctx.materialized_cte_buffer(*id, name)?;
1060+
let scan = Self::new(buffer.id(), name, schema, partitions);
1061+
Ok(Arc::new(scan.bind(buffer)))
1062+
}
8621063
}
8631064

8641065
#[cfg(test)]

‎datafusion/physical-plan/src/proto/mod.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ use datafusion_proto_models::protobuf::{
9393
};
9494

9595
use crate::ExecutionPlan;
96+
use crate::materialized_cte::MaterializedCteBuffer;
9697

9798
pub use datafusion_proto_models::ProtoDecoderRegistry;
9899
// Not public API: see `decode_execution_plan`.
@@ -181,6 +182,16 @@ pub trait ExecutionPlanDecode {
181182

182183
/// Reconstruct a window UDF from its name and optional payload.
183184
fn decode_udwf(&self, name: &str, payload: Option<&[u8]>) -> Result<Arc<WindowUDF>>;
185+
186+
/// The buffer of the materialized CTE that has the id `encoded_id` in the
187+
/// encoded plan. Every call for one id during the decoding of one plan
188+
/// must return the same buffer. See
189+
/// [`DecodedMaterializedCteBuffers`](crate::materialized_cte::DecodedMaterializedCteBuffers).
190+
fn materialized_cte_buffer(
191+
&self,
192+
encoded_id: u64,
193+
name: &str,
194+
) -> Result<Arc<MaterializedCteBuffer>>;
184195
}
185196

186197
/// Context handed to [`ExecutionPlan::try_to_proto`].
@@ -414,6 +425,16 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
414425
self.decoder.decode_udwf(name, payload)
415426
}
416427

428+
/// The buffer that a decoded `MaterializedCteExec` shares with the decoded
429+
/// `MaterializedCteScanExec`s that have the same `encoded_id`.
430+
pub fn materialized_cte_buffer(
431+
&self,
432+
encoded_id: u64,
433+
name: &str,
434+
) -> Result<Arc<MaterializedCteBuffer>> {
435+
self.decoder.materialized_cte_buffer(encoded_id, name)
436+
}
437+
417438
/// An expression-level decode context backed by this plan context, bound to
418439
/// `input_schema`.
419440
///

‎datafusion/physical-plan/src/proto_test_util.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ use datafusion_proto_models::protobuf::{
5454

5555
use crate::ExecutionPlan;
5656
use crate::empty::EmptyExec;
57+
use crate::materialized_cte::MaterializedCteBuffer;
5758
use crate::proto::{ExecutionPlanDecode, ExecutionPlanEncode};
5859

5960
/// The schema shared by the stub child plans: `a: Int32, b: Int32`.
@@ -318,6 +319,14 @@ impl ExecutionPlanDecode for StubPlanDecoder {
318319
fn decode_udwf(&self, name: &str, _payload: Option<&[u8]>) -> Result<Arc<WindowUDF>> {
319320
internal_err!("stub decoder cannot decode the window UDF {name}")
320321
}
322+
323+
fn materialized_cte_buffer(
324+
&self,
325+
_encoded_id: u64,
326+
name: &str,
327+
) -> Result<Arc<MaterializedCteBuffer>> {
328+
internal_err!("stub decoder cannot decode the materialized CTE {name}")
329+
}
321330
}
322331

323332
/// Minimal extension plans for the [`crate::proto::registry`] tests.
@@ -526,4 +535,14 @@ impl ExecutionPlanDecode for UnreachablePlanDecoder {
526535
) -> Result<Arc<WindowUDF>> {
527536
unreachable!("decode_udwf must not be reached when the node is rejected")
528537
}
538+
539+
fn materialized_cte_buffer(
540+
&self,
541+
_encoded_id: u64,
542+
_name: &str,
543+
) -> Result<Arc<MaterializedCteBuffer>> {
544+
unreachable!(
545+
"materialized_cte_buffer must not be reached when the node is rejected"
546+
)
547+
}
529548
}

0 commit comments

Comments
 (0)