Skip to content

Commit 6f12a16

Browse files
committed
refactor(proto): migrate scalar subquery serde
ScalarSubqueryExpr nodes must share the results container populated by their enclosing ScalarSubqueryExec. A normal child decode loses that scope when serialization moves into the plan implementation. Add a scoped child decode operation so the new plan hooks preserve the shared container while removing the central serialization path. Closes apache#23515 Signed-off-by: Jiawei Zhao <[email protected]>
1 parent e8a65f2 commit 6f12a16

3 files changed

Lines changed: 120 additions & 60 deletions

File tree

‎datafusion/physical-plan/src/proto.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ use std::sync::Arc;
6262
use arrow::datatypes::Schema;
6363
use datafusion_common::{Result, internal_datafusion_err};
6464
use datafusion_execution::TaskContext;
65+
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
6566
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
6667
use datafusion_physical_expr::PhysicalExpr;
6768
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
@@ -102,6 +103,14 @@ pub trait ExecutionPlanDecode {
102103
/// deserializer, so the child's own `try_from_proto` is honored).
103104
fn decode_plan(&self, node: &PhysicalPlanNode) -> Result<Arc<dyn ExecutionPlan>>;
104105

106+
/// Deserialize a child plan with `results` active for scalar subquery
107+
/// expressions in that plan's subtree.
108+
fn decode_plan_with_scalar_subquery_results(
109+
&self,
110+
node: &PhysicalPlanNode,
111+
results: ScalarSubqueryResults,
112+
) -> Result<Arc<dyn ExecutionPlan>>;
113+
105114
/// Deserialize a physical expression against `input_schema`.
106115
fn decode_expr(
107116
&self,
@@ -215,6 +224,17 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
215224
self.decoder.decode_plan(node)
216225
}
217226

227+
/// Deserialize a child plan with `results` active for scalar subquery
228+
/// expressions in that plan's subtree.
229+
pub fn decode_child_with_scalar_subquery_results(
230+
&self,
231+
node: &PhysicalPlanNode,
232+
results: ScalarSubqueryResults,
233+
) -> Result<Arc<dyn ExecutionPlan>> {
234+
self.decoder
235+
.decode_plan_with_scalar_subquery_results(node, results)
236+
}
237+
218238
/// Deserialize a required child plan, producing a uniform "missing required
219239
/// field" error when the optional wire field is absent.
220240
pub fn decode_required_child(

‎datafusion/physical-plan/src/scalar_subquery.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,68 @@ impl ExecutionPlan for ScalarSubqueryExec {
254254
fn cardinality_effect(&self) -> CardinalityEffect {
255255
CardinalityEffect::Equal
256256
}
257+
258+
#[cfg(feature = "proto")]
259+
fn try_to_proto(
260+
&self,
261+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
262+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
263+
use datafusion_proto_models::protobuf;
264+
265+
let input = ctx.encode_child(self.input())?;
266+
// Subquery indices are positional and recovered during decoding.
267+
let subqueries =
268+
ctx.encode_children(self.subqueries().iter().map(|subquery| &subquery.plan))?;
269+
Ok(Some(protobuf::PhysicalPlanNode {
270+
physical_plan_type: Some(
271+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new(
272+
protobuf::ScalarSubqueryExecNode {
273+
input: Some(Box::new(input)),
274+
subqueries,
275+
},
276+
)),
277+
),
278+
}))
279+
}
280+
}
281+
282+
#[cfg(feature = "proto")]
283+
impl ScalarSubqueryExec {
284+
/// Reconstruct a [`ScalarSubqueryExec`] from its protobuf representation.
285+
pub fn try_from_proto(
286+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
287+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
288+
) -> Result<Arc<dyn ExecutionPlan>> {
289+
use datafusion_proto_models::protobuf;
290+
291+
let scalar_subquery = crate::expect_plan_variant!(
292+
node,
293+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery,
294+
"ScalarSubqueryExec",
295+
);
296+
let results = ScalarSubqueryResults::new(scalar_subquery.subqueries.len());
297+
let input_node = scalar_subquery.input.as_deref().ok_or_else(|| {
298+
datafusion_common::internal_datafusion_err!(
299+
"ScalarSubqueryExec is missing required field 'input'"
300+
)
301+
})?;
302+
// The input's ScalarSubqueryExpr nodes must share this results container.
303+
let input =
304+
ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?;
305+
let subqueries = scalar_subquery
306+
.subqueries
307+
.iter()
308+
.enumerate()
309+
.map(|(index, plan)| {
310+
Ok(ScalarSubqueryLink {
311+
plan: ctx.decode_child(plan)?,
312+
index: SubqueryIndex::new(index),
313+
})
314+
})
315+
.collect::<Result<Vec<_>>>()?;
316+
317+
Ok(Arc::new(Self::new(input, subqueries, results)))
318+
}
257319
}
258320

259321
/// Wait for the subquery execution future to complete.

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 38 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ use datafusion_datasource_parquet::source::ParquetSource;
4949
#[cfg(feature = "parquet")]
5050
use datafusion_execution::object_store::ObjectStoreUrl;
5151
use datafusion_execution::{FunctionRegistry, TaskContext};
52-
use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};
52+
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
5353
use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
5454
use datafusion_functions_table::generate_series::{
5555
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
@@ -85,7 +85,7 @@ use datafusion_physical_plan::proto::{
8585
ExecutionPlanEncodeCtx,
8686
};
8787
use datafusion_physical_plan::repartition::RepartitionExec;
88-
use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};
88+
use datafusion_physical_plan::scalar_subquery::ScalarSubqueryExec;
8989
use datafusion_physical_plan::sorts::sort::SortExec;
9090
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
9191
use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
@@ -811,8 +811,8 @@ pub trait PhysicalPlanNodeExt: Sized {
811811
PhysicalPlanType::Buffer(_) => {
812812
BufferExec::try_from_proto(self.node(), &decode_ctx)
813813
}
814-
PhysicalPlanType::ScalarSubquery(sq) => {
815-
self.try_into_scalar_subquery_physical_plan(sq, ctx, proto_converter)
814+
PhysicalPlanType::ScalarSubquery(_) => {
815+
ScalarSubqueryExec::try_from_proto(self.node(), &decode_ctx)
816816
}
817817
}
818818
}
@@ -872,14 +872,6 @@ pub trait PhysicalPlanNodeExt: Sized {
872872
return Ok(node);
873873
}
874874

875-
if let Some(exec) = plan.downcast_ref::<ScalarSubqueryExec>() {
876-
return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec(
877-
exec,
878-
codec,
879-
proto_converter,
880-
);
881-
}
882-
883875
let mut buf: Vec<u8> = vec![];
884876
match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {
885877
Ok(_) => {
@@ -1950,38 +1942,27 @@ pub trait PhysicalPlanNodeExt: Sized {
19501942
BufferExec::try_from_proto(&node, &decode_ctx)
19511943
}
19521944

1945+
#[deprecated(
1946+
since = "55.0.0",
1947+
note = "unused by DataFusion; `ScalarSubqueryExec` deserializes itself via `ScalarSubqueryExec::try_from_proto`"
1948+
)]
19531949
fn try_into_scalar_subquery_physical_plan(
19541950
&self,
19551951
sq: &protobuf::ScalarSubqueryExecNode,
19561952
ctx: &PhysicalPlanDecodeContext<'_>,
19571953
proto_converter: &dyn PhysicalProtoConverterExtension,
19581954
) -> Result<Arc<dyn ExecutionPlan>> {
1959-
// First, deserialize the main input plan. We set up the subquery results
1960-
// container first, so that ScalarSubqueryExpr nodes can reference it.
1961-
let subquery_results = ScalarSubqueryResults::new(sq.subqueries.len());
1962-
let input_ctx = ctx.with_scalar_subquery_results(subquery_results.clone());
1963-
let input = into_physical_plan(&sq.input, &input_ctx, proto_converter)?;
1964-
1965-
// Now deserialize the subquery children.
1966-
let subqueries: Vec<ScalarSubqueryLink> = sq
1967-
.subqueries
1968-
.iter()
1969-
.enumerate()
1970-
.map(|(index, sq_plan)| {
1971-
let plan =
1972-
sq_plan.try_into_physical_plan_with_context(ctx, proto_converter)?;
1973-
Ok(ScalarSubqueryLink {
1974-
plan,
1975-
index: SubqueryIndex::new(index),
1976-
})
1977-
})
1978-
.collect::<Result<Vec<_>>>()?;
1979-
1980-
Ok(Arc::new(ScalarSubqueryExec::new(
1981-
input,
1982-
subqueries,
1983-
subquery_results,
1984-
)))
1955+
let node = protobuf::PhysicalPlanNode {
1956+
physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new(
1957+
sq.clone(),
1958+
))),
1959+
};
1960+
let decoder = ConverterPlanDecoder {
1961+
ctx,
1962+
proto_converter,
1963+
};
1964+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1965+
ScalarSubqueryExec::try_from_proto(&node, &decode_ctx)
19851966
}
19861967

19871968
#[deprecated(
@@ -2866,35 +2847,22 @@ pub trait PhysicalPlanNodeExt: Sized {
28662847
.ok_or_else(|| internal_datafusion_err!("BufferExec is not serializable"))
28672848
}
28682849

2850+
#[deprecated(
2851+
since = "55.0.0",
2852+
note = "unused by DataFusion; `ScalarSubqueryExec` serializes itself via `ExecutionPlan::try_to_proto`"
2853+
)]
28692854
fn try_from_scalar_subquery_exec(
28702855
exec: &ScalarSubqueryExec,
28712856
codec: &dyn PhysicalExtensionCodec,
28722857
proto_converter: &dyn PhysicalProtoConverterExtension,
28732858
) -> Result<protobuf::PhysicalPlanNode> {
2874-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2875-
Arc::clone(exec.input()),
2859+
let encoder = ConverterPlanEncoder {
28762860
codec,
28772861
proto_converter,
2878-
)?;
2879-
let subqueries = exec
2880-
.subqueries()
2881-
.iter()
2882-
.map(|sq| {
2883-
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2884-
Arc::clone(&sq.plan),
2885-
codec,
2886-
proto_converter,
2887-
)
2888-
})
2889-
.collect::<Result<Vec<_>>>()?;
2890-
2891-
Ok(protobuf::PhysicalPlanNode {
2892-
physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new(
2893-
protobuf::ScalarSubqueryExecNode {
2894-
input: Some(Box::new(input)),
2895-
subqueries,
2896-
},
2897-
))),
2862+
};
2863+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
2864+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2865+
internal_datafusion_err!("ScalarSubqueryExec is not serializable")
28982866
})
28992867
}
29002868
}
@@ -3485,6 +3453,16 @@ impl ExecutionPlanDecode for ConverterPlanDecoder<'_, '_> {
34853453
self.proto_converter.proto_to_execution_plan(node, self.ctx)
34863454
}
34873455

3456+
fn decode_plan_with_scalar_subquery_results(
3457+
&self,
3458+
node: &protobuf::PhysicalPlanNode,
3459+
results: ScalarSubqueryResults,
3460+
) -> Result<Arc<dyn ExecutionPlan>> {
3461+
let scoped_ctx = self.ctx.with_scalar_subquery_results(results);
3462+
self.proto_converter
3463+
.proto_to_execution_plan(node, &scoped_ctx)
3464+
}
3465+
34883466
fn decode_expr(
34893467
&self,
34903468
node: &protobuf::PhysicalExprNode,

0 commit comments

Comments
 (0)