Skip to content

Commit 15ec040

Browse files
adriangbclaude
andcommitted
prototype: migrate AsyncFuncExec + ScalarSubqueryExec onto the proto hook (apache#22419)
AsyncFuncExec: clean single-child + N-exprs, no codec. ScalarSubqueryExec needed one small typed ctx addition: decode_child_with_scalar_subquery_results (dispatch method + ctx wrapper + one-line datafusion-proto adapter that re-enters with a scalar-subquery-results scope). The scope handle is a datafusion-expr type, so no proto type leaks and no codec is exposed — same precedent as the UDF ctx methods. The existing roundtrip_scalar_subquery_exec test exercises the scoped decode. 192 roundtrip tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 5f60509 commit 15ec040

4 files changed

Lines changed: 171 additions & 20 deletions

File tree

‎datafusion/physical-plan/src/async_func.rs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,80 @@ impl ExecutionPlan for AsyncFuncExec {
246246
fn metrics(&self) -> Option<MetricsSet> {
247247
Some(self.metrics.clone_inner())
248248
}
249+
250+
#[cfg(feature = "proto")]
251+
fn try_to_proto(
252+
&self,
253+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
254+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
255+
use datafusion_proto_models::protobuf;
256+
let input = ctx.encode_child(self.input())?;
257+
let async_exprs =
258+
ctx.encode_expressions(self.async_exprs.iter().map(|e| &e.func))?;
259+
let async_expr_names = self.async_exprs.iter().map(|e| e.name.clone()).collect();
260+
Ok(Some(protobuf::PhysicalPlanNode {
261+
physical_plan_type: Some(
262+
protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc(Box::new(
263+
protobuf::AsyncFuncExecNode {
264+
input: Some(Box::new(input)),
265+
async_exprs,
266+
async_expr_names,
267+
},
268+
)),
269+
),
270+
}))
271+
}
272+
}
273+
274+
#[cfg(feature = "proto")]
275+
impl AsyncFuncExec {
276+
/// Reconstruct an [`AsyncFuncExec`] from its protobuf representation.
277+
///
278+
/// The exact inverse of [`ExecutionPlan::try_to_proto`], taking the whole
279+
/// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one
280+
/// signature.
281+
///
282+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
283+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
284+
pub fn try_from_proto(
285+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
286+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
287+
) -> Result<Arc<dyn ExecutionPlan>> {
288+
use datafusion_proto_models::protobuf;
289+
let async_func = crate::expect_plan_variant!(
290+
node,
291+
protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc,
292+
"AsyncFuncExec",
293+
);
294+
let input = ctx.decode_required_child(
295+
async_func.input.as_deref(),
296+
"AsyncFuncExec",
297+
"input",
298+
)?;
299+
300+
if async_func.async_exprs.len() != async_func.async_expr_names.len() {
301+
return datafusion_common::internal_err!(
302+
"AsyncFuncExecNode async_exprs length does not match async_expr_names"
303+
);
304+
}
305+
306+
let input_schema = input.schema();
307+
let async_exprs = async_func
308+
.async_exprs
309+
.iter()
310+
.zip(async_func.async_expr_names.iter())
311+
.map(|(expr, name)| {
312+
let physical_expr = ctx.decode_expr(expr, input_schema.as_ref())?;
313+
Ok(Arc::new(AsyncFuncExpr::try_new(
314+
name.clone(),
315+
physical_expr,
316+
input_schema.as_ref(),
317+
)?))
318+
})
319+
.collect::<Result<Vec<_>>>()?;
320+
321+
Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?))
322+
}
249323
}
250324

251325
struct CoalesceInputStream {

‎datafusion/physical-plan/src/proto.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ use std::sync::Arc;
6262
use arrow::datatypes::Schema;
6363
use datafusion_common::{Result, internal_datafusion_err};
6464
use datafusion_execution::TaskContext;
65+
use datafusion_expr::execution_props::ScalarSubqueryResults;
6566
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
6667
use datafusion_physical_expr::PhysicalExpr;
6768
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
@@ -109,6 +110,15 @@ pub trait ExecutionPlanDecode {
109110
input_schema: &Schema,
110111
) -> Result<Arc<dyn PhysicalExpr>>;
111112

113+
/// Deserialize a child plan within a scope where `results` is the active
114+
/// scalar-subquery-results container, so `ScalarSubqueryExpr` nodes anywhere
115+
/// in the child subtree resolve against it. Used only by `ScalarSubqueryExec`.
116+
fn decode_plan_with_scalar_subquery_results(
117+
&self,
118+
node: &PhysicalPlanNode,
119+
results: ScalarSubqueryResults,
120+
) -> Result<Arc<dyn ExecutionPlan>>;
121+
112122
/// The session task context, used by plans that need the function registry
113123
/// or session configuration. Never exposes the proto extension codec.
114124
fn task_ctx(&self) -> &TaskContext;
@@ -225,6 +235,18 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
225235
nodes.into_iter().map(|n| self.decode_child(n)).collect()
226236
}
227237

238+
/// Deserialize a child plan within a scope carrying `results` as the active
239+
/// scalar-subquery-results container (used only by `ScalarSubqueryExec`, so
240+
/// that `ScalarSubqueryExpr` nodes in the child subtree resolve against it).
241+
pub fn decode_child_with_scalar_subquery_results(
242+
&self,
243+
node: &PhysicalPlanNode,
244+
results: ScalarSubqueryResults,
245+
) -> Result<Arc<dyn ExecutionPlan>> {
246+
self.decoder
247+
.decode_plan_with_scalar_subquery_results(node, results)
248+
}
249+
228250
/// Deserialize a required child plan, producing a uniform "missing required
229251
/// field" error when the optional wire field is absent.
230252
pub fn decode_required_child(

‎datafusion/physical-plan/src/scalar_subquery.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,68 @@ impl ExecutionPlan for ScalarSubqueryExec {
254254
fn cardinality_effect(&self) -> CardinalityEffect {
255255
CardinalityEffect::Equal
256256
}
257+
258+
#[cfg(feature = "proto")]
259+
fn try_to_proto(
260+
&self,
261+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
262+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
263+
use datafusion_proto_models::protobuf;
264+
let input = ctx.encode_child(self.input())?;
265+
// Index is positional (recovered on decode via enumerate), not on the wire.
266+
let subqueries =
267+
ctx.encode_children(self.subqueries().iter().map(|sq| &sq.plan))?;
268+
Ok(Some(protobuf::PhysicalPlanNode {
269+
physical_plan_type: Some(
270+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new(
271+
protobuf::ScalarSubqueryExecNode {
272+
input: Some(Box::new(input)),
273+
subqueries,
274+
},
275+
)),
276+
),
277+
}))
278+
}
279+
}
280+
281+
#[cfg(feature = "proto")]
282+
impl ScalarSubqueryExec {
283+
/// Reconstruct a [`ScalarSubqueryExec`] from its protobuf representation.
284+
pub fn try_from_proto(
285+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
286+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
287+
) -> Result<Arc<dyn ExecutionPlan>> {
288+
use datafusion_proto_models::protobuf;
289+
let sq = crate::expect_plan_variant!(
290+
node,
291+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery,
292+
"ScalarSubqueryExec",
293+
);
294+
// The subquery-results container must be active while decoding the input,
295+
// so ScalarSubqueryExpr nodes in the input subtree resolve against it.
296+
let results = ScalarSubqueryResults::new(sq.subqueries.len());
297+
let input_node = sq.input.as_deref().ok_or_else(|| {
298+
datafusion_common::internal_datafusion_err!(
299+
"ScalarSubqueryExec is missing required field 'input'"
300+
)
301+
})?;
302+
let input =
303+
ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?;
304+
let subqueries = sq
305+
.subqueries
306+
.iter()
307+
.enumerate()
308+
.map(|(index, plan_node)| {
309+
Ok(ScalarSubqueryLink {
310+
plan: ctx.decode_child(plan_node)?,
311+
index: SubqueryIndex::new(index),
312+
})
313+
})
314+
.collect::<Result<Vec<_>>>()?;
315+
Ok(Arc::new(ScalarSubqueryExec::new(
316+
input, subqueries, results,
317+
)))
318+
}
257319
}
258320

259321
/// Wait for the subquery execution future to complete.

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -866,14 +866,14 @@ pub trait PhysicalPlanNodeExt: Sized {
866866
PhysicalPlanType::SortMergeJoin(_) => {
867867
SortMergeJoinExec::try_from_proto(self.node(), &decode_ctx)
868868
}
869-
PhysicalPlanType::AsyncFunc(async_func) => {
870-
self.try_into_async_func_physical_plan(async_func, ctx, proto_converter)
869+
PhysicalPlanType::AsyncFunc(_) => {
870+
AsyncFuncExec::try_from_proto(self.node(), &decode_ctx)
871871
}
872872
PhysicalPlanType::Buffer(_) => {
873873
BufferExec::try_from_proto(self.node(), &decode_ctx)
874874
}
875-
PhysicalPlanType::ScalarSubquery(sq) => {
876-
self.try_into_scalar_subquery_physical_plan(sq, ctx, proto_converter)
875+
PhysicalPlanType::ScalarSubquery(_) => {
876+
ScalarSubqueryExec::try_from_proto(self.node(), &decode_ctx)
877877
}
878878
}
879879
}
@@ -949,22 +949,6 @@ pub trait PhysicalPlanNodeExt: Sized {
949949
return Ok(node);
950950
}
951951

952-
if let Some(exec) = plan.downcast_ref::<AsyncFuncExec>() {
953-
return protobuf::PhysicalPlanNode::try_from_async_func_exec(
954-
exec,
955-
codec,
956-
proto_converter,
957-
);
958-
}
959-
960-
if let Some(exec) = plan.downcast_ref::<ScalarSubqueryExec>() {
961-
return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec(
962-
exec,
963-
codec,
964-
proto_converter,
965-
);
966-
}
967-
968952
let mut buf: Vec<u8> = vec![];
969953
match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {
970954
Ok(_) => {
@@ -4647,6 +4631,15 @@ impl ExecutionPlanDecode for ConverterPlanDecoder<'_, '_> {
46474631
.proto_to_physical_expr(node, input_schema, self.ctx)
46484632
}
46494633

4634+
fn decode_plan_with_scalar_subquery_results(
4635+
&self,
4636+
node: &protobuf::PhysicalPlanNode,
4637+
results: ScalarSubqueryResults,
4638+
) -> Result<Arc<dyn ExecutionPlan>> {
4639+
let scoped = self.ctx.with_scalar_subquery_results(results);
4640+
self.proto_converter.proto_to_execution_plan(node, &scoped)
4641+
}
4642+
46504643
fn task_ctx(&self) -> &TaskContext {
46514644
self.ctx.task_ctx()
46524645
}

0 commit comments

Comments
 (0)