Skip to content

Commit 1388fb1

Browse files
adriangbclaude
andcommitted
prototype: ExecutionPlan try_to_proto/try_from_proto hook + ProjectionExec reference (apache#22419)
General part: ExecutionPlanEncodeCtx/DecodeCtx + internal dispatch traits in datafusion-physical-plan (mirrors the PhysicalExpr pattern), the try_to_proto trait hook, expect_plan_variant! macro, and the ConverterPlanEncoder/Decoder adapters wiring the hook into the central dispatch in datafusion-proto. ProjectionExec migrated as the reference: encode downcast arm deleted, decode arm reduced to a one-liner delegating to ProjectionExec::try_from_proto. All 189 proto_integration roundtrip tests pass (incl. TPC-H). Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 95de385 commit 1388fb1

5 files changed

Lines changed: 402 additions & 74 deletions

File tree

‎datafusion/physical-plan/src/execution_plan.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,27 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
823823
) -> Option<Arc<dyn ExecutionPlan>> {
824824
None
825825
}
826+
827+
/// Serialize this plan to its protobuf representation, if it knows how.
828+
///
829+
/// This is the `ExecutionPlan` analog of
830+
/// [`PhysicalExpr::try_to_proto`](datafusion_physical_expr_common::physical_expr::PhysicalExpr::try_to_proto).
831+
///
832+
/// * `Ok(None)` (the default) — "I don't serialize myself"; the caller
833+
/// (`datafusion-proto`) falls back to the central downcast chain. Every
834+
/// un-migrated plan keeps its existing behavior.
835+
/// * `Ok(Some(node))` — fully serialized; the caller must not fall back.
836+
/// * `Err(_)` — a real failure (e.g. a child failed to serialize).
837+
///
838+
/// Only *self-contained* plans should override this — see [`crate::proto`]
839+
/// for the session-dependency boundary.
840+
#[cfg(feature = "proto")]
841+
fn try_to_proto(
842+
&self,
843+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
844+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
845+
Ok(None)
846+
}
826847
}
827848

828849
impl dyn ExecutionPlan {

‎datafusion/physical-plan/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ pub mod metrics;
8888
pub mod operator_statistics;
8989
pub mod placeholder_row;
9090
pub mod projection;
91+
#[cfg(feature = "proto")]
92+
pub mod proto;
9193
pub mod recursive_query;
9294
pub mod repartition;
9395
pub mod scalar_subquery;

‎datafusion/physical-plan/src/projection.rs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,71 @@ impl ExecutionPlan for ProjectionExec {
500500
.ok()
501501
})
502502
}
503+
504+
#[cfg(feature = "proto")]
505+
fn try_to_proto(
506+
&self,
507+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
508+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
509+
use datafusion_proto_models::protobuf;
510+
let input = ctx.encode_child(self.input())?;
511+
let expr = ctx.encode_expressions(self.expr().iter().map(|p| &p.expr))?;
512+
let expr_name = self.expr().iter().map(|p| p.alias.clone()).collect();
513+
Ok(Some(protobuf::PhysicalPlanNode {
514+
physical_plan_type: Some(
515+
protobuf::physical_plan_node::PhysicalPlanType::Projection(Box::new(
516+
protobuf::ProjectionExecNode {
517+
input: Some(Box::new(input)),
518+
expr,
519+
expr_name,
520+
},
521+
)),
522+
),
523+
}))
524+
}
525+
}
526+
527+
#[cfg(feature = "proto")]
528+
impl ProjectionExec {
529+
/// Reconstruct a [`ProjectionExec`] from its protobuf representation.
530+
///
531+
/// The exact inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
532+
/// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one
533+
/// signature. Child plans and expressions are decoded recursively via the
534+
/// [`ExecutionPlanDecodeCtx`].
535+
///
536+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
537+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
538+
/// [`ExecutionPlanDecodeCtx`]: crate::proto::ExecutionPlanDecodeCtx
539+
pub fn try_from_proto(
540+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
541+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
542+
) -> Result<Arc<dyn ExecutionPlan>> {
543+
use datafusion_proto_models::protobuf;
544+
let projection = crate::expect_plan_variant!(
545+
node,
546+
protobuf::physical_plan_node::PhysicalPlanType::Projection,
547+
"ProjectionExec",
548+
);
549+
let input = ctx.decode_required_child(
550+
projection.input.as_deref(),
551+
"ProjectionExec",
552+
"input",
553+
)?;
554+
let input_schema = input.schema();
555+
let exprs = projection
556+
.expr
557+
.iter()
558+
.zip(projection.expr_name.iter())
559+
.map(|(expr, name)| {
560+
Ok(ProjectionExpr {
561+
expr: ctx.decode_expr(expr, input_schema.as_ref())?,
562+
alias: name.to_string(),
563+
})
564+
})
565+
.collect::<Result<Vec<ProjectionExpr>>>()?;
566+
Ok(Arc::new(ProjectionExec::try_new(exprs, input)?))
567+
}
503568
}
504569

505570
impl ProjectionStream {
Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Serialization hooks for [`ExecutionPlan`], mirroring the
19+
//! `try_to_proto`/`try_from_proto` pattern used for `PhysicalExpr`.
20+
//!
21+
//! # Why the indirection
22+
//!
23+
//! An `ExecutionPlan` must be able to (de)serialize its child plans and its
24+
//! child physical expressions recursively. The concrete recursion lives in
25+
//! `datafusion-proto` (it owns the extension codec, the session context and the
26+
//! central converter), but `datafusion-proto` sits *above* `datafusion-physical-plan`
27+
//! in the crate graph. To let a plan drive that recursion without a dependency
28+
//! cycle, this module defines:
29+
//!
30+
//! * [`ExecutionPlanEncodeCtx`] / [`ExecutionPlanDecodeCtx`] — the stable,
31+
//! concrete context types a plan author interacts with. New capabilities can
32+
//! be added here without changing every plan's hook signature.
33+
//! * [`ExecutionPlanEncode`] / [`ExecutionPlanDecode`] — internal dispatch
34+
//! traits, *defined* here but *implemented* in `datafusion-proto`, that the
35+
//! context types delegate to. This is the dependency inversion that keeps the
36+
//! proto types flowing in one direction only.
37+
//!
38+
//! `datafusion-physical-plan` depends on the pure prost types in
39+
//! `datafusion-proto-models` (feature `proto`), never on `datafusion-proto`.
40+
//!
41+
//! # Scope: self-contained plans only
42+
//!
43+
//! These hooks are for plans that are *self-contained*: everything they need to
44+
//! round-trip is reachable from their child plans, their child expressions and
45+
//! their input schema. Plans that reference session-scoped machinery only
46+
//! `datafusion-proto` owns — the [`PhysicalExtensionCodec`] (for UDAF/UDWF and
47+
//! extension nodes) — stay as typed dispatch arms inside `datafusion-proto`,
48+
//! exactly as `ScalarFunctionExpr` does on the expression side. The decode
49+
//! context intentionally exposes the [`TaskContext`] (for function-registry and
50+
//! session-config lookups reachable from `datafusion-execution`) but *not* the
51+
//! extension codec.
52+
//!
53+
//! [`PhysicalExtensionCodec`]: (owned by `datafusion-proto`)
54+
//! [`ExecutionPlan`]: crate::ExecutionPlan
55+
56+
use std::sync::Arc;
57+
58+
use arrow::datatypes::Schema;
59+
use datafusion_common::{Result, internal_datafusion_err};
60+
use datafusion_execution::TaskContext;
61+
use datafusion_physical_expr::PhysicalExpr;
62+
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
63+
64+
use crate::ExecutionPlan;
65+
66+
/// Internal dispatch trait backing [`ExecutionPlanEncodeCtx`].
67+
///
68+
/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
69+
/// call methods on [`ExecutionPlanEncodeCtx`] instead.
70+
pub trait ExecutionPlanEncode {
71+
/// Serialize a child execution plan (recursing through the central
72+
/// serializer, so the child's own `try_to_proto` hook is honored).
73+
fn encode_plan(&self, plan: &Arc<dyn ExecutionPlan>) -> Result<PhysicalPlanNode>;
74+
75+
/// Serialize a physical expression owned by the plan.
76+
fn encode_expr(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode>;
77+
}
78+
79+
/// Internal dispatch trait backing [`ExecutionPlanDecodeCtx`].
80+
///
81+
/// Implemented by `datafusion-proto`. Plan authors never name this trait; they
82+
/// call methods on [`ExecutionPlanDecodeCtx`] instead.
83+
pub trait ExecutionPlanDecode {
84+
/// Deserialize a child execution plan (recursing through the central
85+
/// deserializer, so the child's own `try_from_proto` is honored).
86+
fn decode_plan(&self, node: &PhysicalPlanNode) -> Result<Arc<dyn ExecutionPlan>>;
87+
88+
/// Deserialize a physical expression against `input_schema`.
89+
fn decode_expr(
90+
&self,
91+
node: &PhysicalExprNode,
92+
input_schema: &Schema,
93+
) -> Result<Arc<dyn PhysicalExpr>>;
94+
95+
/// The session task context, used by plans that need the function registry
96+
/// or session configuration. Never exposes the proto extension codec.
97+
fn task_ctx(&self) -> &TaskContext;
98+
}
99+
100+
/// Context handed to [`ExecutionPlan::try_to_proto`](crate::ExecutionPlan::try_to_proto).
101+
///
102+
/// Provides the primitives a plan needs to serialize its children and
103+
/// expressions without naming `datafusion-proto`.
104+
pub struct ExecutionPlanEncodeCtx<'a> {
105+
encoder: &'a dyn ExecutionPlanEncode,
106+
}
107+
108+
impl<'a> ExecutionPlanEncodeCtx<'a> {
109+
/// Create a new encode context wrapping an [`ExecutionPlanEncode`]
110+
/// implementation (supplied by `datafusion-proto`).
111+
pub fn new(encoder: &'a dyn ExecutionPlanEncode) -> Self {
112+
Self { encoder }
113+
}
114+
115+
/// Serialize a single child plan.
116+
pub fn encode_child(
117+
&self,
118+
plan: &Arc<dyn ExecutionPlan>,
119+
) -> Result<PhysicalPlanNode> {
120+
self.encoder.encode_plan(plan)
121+
}
122+
123+
/// Serialize an iterator of child plans.
124+
pub fn encode_children<'b, I>(&self, plans: I) -> Result<Vec<PhysicalPlanNode>>
125+
where
126+
I: IntoIterator<Item = &'b Arc<dyn ExecutionPlan>>,
127+
{
128+
plans.into_iter().map(|p| self.encode_child(p)).collect()
129+
}
130+
131+
/// Serialize a single physical expression.
132+
pub fn encode_expr(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<PhysicalExprNode> {
133+
self.encoder.encode_expr(expr)
134+
}
135+
136+
/// Serialize an iterator of physical expressions.
137+
pub fn encode_expressions<'b, I>(&self, exprs: I) -> Result<Vec<PhysicalExprNode>>
138+
where
139+
I: IntoIterator<Item = &'b Arc<dyn PhysicalExpr>>,
140+
{
141+
exprs.into_iter().map(|e| self.encode_expr(e)).collect()
142+
}
143+
}
144+
145+
/// Context handed to a plan's `try_from_proto` associated function.
146+
///
147+
/// Provides the primitives a plan needs to deserialize its children and
148+
/// expressions without naming `datafusion-proto`.
149+
pub struct ExecutionPlanDecodeCtx<'a> {
150+
decoder: &'a dyn ExecutionPlanDecode,
151+
}
152+
153+
impl<'a> ExecutionPlanDecodeCtx<'a> {
154+
/// Create a new decode context wrapping an [`ExecutionPlanDecode`]
155+
/// implementation (supplied by `datafusion-proto`).
156+
pub fn new(decoder: &'a dyn ExecutionPlanDecode) -> Self {
157+
Self { decoder }
158+
}
159+
160+
/// Deserialize a single child plan.
161+
pub fn decode_child(
162+
&self,
163+
node: &PhysicalPlanNode,
164+
) -> Result<Arc<dyn ExecutionPlan>> {
165+
self.decoder.decode_plan(node)
166+
}
167+
168+
/// Deserialize a required child plan, producing a uniform "missing required
169+
/// field" error when the optional wire field is absent.
170+
pub fn decode_required_child(
171+
&self,
172+
node: Option<&PhysicalPlanNode>,
173+
plan_name: &str,
174+
field: &str,
175+
) -> Result<Arc<dyn ExecutionPlan>> {
176+
let node = node.ok_or_else(|| {
177+
internal_datafusion_err!("{plan_name} is missing required field '{field}'")
178+
})?;
179+
self.decode_child(node)
180+
}
181+
182+
/// Deserialize a physical expression against `input_schema`.
183+
pub fn decode_expr(
184+
&self,
185+
node: &PhysicalExprNode,
186+
input_schema: &Schema,
187+
) -> Result<Arc<dyn PhysicalExpr>> {
188+
self.decoder.decode_expr(node, input_schema)
189+
}
190+
191+
/// Deserialize a required physical expression against `input_schema`.
192+
pub fn decode_required_expr(
193+
&self,
194+
node: Option<&PhysicalExprNode>,
195+
input_schema: &Schema,
196+
plan_name: &str,
197+
field: &str,
198+
) -> Result<Arc<dyn PhysicalExpr>> {
199+
let node = node.ok_or_else(|| {
200+
internal_datafusion_err!("{plan_name} is missing required field '{field}'")
201+
})?;
202+
self.decode_expr(node, input_schema)
203+
}
204+
205+
/// The session task context (function registry + session config). Never
206+
/// exposes the proto extension codec — session-codec-dependent plans stay as
207+
/// typed dispatch arms in `datafusion-proto`.
208+
pub fn task_ctx(&self) -> &TaskContext {
209+
self.decoder.task_ctx()
210+
}
211+
}
212+
213+
/// Assert that a [`PhysicalPlanNode`] carries the expected `PhysicalPlanType`
214+
/// variant, returning a reference to the inner payload, else an `internal_err!`.
215+
/// Mirrors `expect_expr_variant!` on the expression side. Field access on the
216+
/// result auto-derefs through the `Box` that boxed variants use.
217+
#[macro_export]
218+
macro_rules! expect_plan_variant {
219+
($node:expr, $variant:path, $plan_name:literal $(,)?) => {{
220+
match &$node.physical_plan_type {
221+
Some($variant(inner)) => inner,
222+
_ => {
223+
return ::datafusion_common::internal_err!(concat!(
224+
"PhysicalPlanNode is not a ",
225+
$plan_name
226+
));
227+
}
228+
}
229+
}};
230+
}

0 commit comments

Comments
 (0)