Skip to content

Commit 3aef16c

Browse files
refactor(proto): migrate scalar subquery serde (apache#23915)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Closes apache#23515. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ScalarSubqueryExpr nodes must share the results container populated by their enclosing ScalarSubqueryExec. A normal child decode loses that scope when serialization moves into the plan implementation. Add a scoped child decode operation so the new plan hooks preserve the shared container while removing the central serialization path. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> Signed-off-by: Jiawei Zhao <[email protected]>
1 parent 62cfc0c commit 3aef16c

3 files changed

Lines changed: 120 additions & 60 deletions

File tree

‎datafusion/physical-plan/src/proto.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ use std::sync::Arc;
6262
use arrow::datatypes::Schema;
6363
use datafusion_common::{Result, internal_datafusion_err};
6464
use datafusion_execution::TaskContext;
65+
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
6566
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
6667
use datafusion_physical_expr::PhysicalExpr;
6768
use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode};
@@ -102,6 +103,14 @@ pub trait ExecutionPlanDecode {
102103
/// deserializer, so the child's own `try_from_proto` is honored).
103104
fn decode_plan(&self, node: &PhysicalPlanNode) -> Result<Arc<dyn ExecutionPlan>>;
104105

106+
/// Deserialize a child plan with `results` active for scalar subquery
107+
/// expressions in that plan's subtree.
108+
fn decode_plan_with_scalar_subquery_results(
109+
&self,
110+
node: &PhysicalPlanNode,
111+
results: ScalarSubqueryResults,
112+
) -> Result<Arc<dyn ExecutionPlan>>;
113+
105114
/// Deserialize a physical expression against `input_schema`.
106115
fn decode_expr(
107116
&self,
@@ -215,6 +224,17 @@ impl<'a> ExecutionPlanDecodeCtx<'a> {
215224
self.decoder.decode_plan(node)
216225
}
217226

227+
/// Deserialize a child plan with `results` active for scalar subquery
228+
/// expressions in that plan's subtree.
229+
pub fn decode_child_with_scalar_subquery_results(
230+
&self,
231+
node: &PhysicalPlanNode,
232+
results: ScalarSubqueryResults,
233+
) -> Result<Arc<dyn ExecutionPlan>> {
234+
self.decoder
235+
.decode_plan_with_scalar_subquery_results(node, results)
236+
}
237+
218238
/// Deserialize a required child plan, producing a uniform "missing required
219239
/// field" error when the optional wire field is absent.
220240
pub fn decode_required_child(

‎datafusion/physical-plan/src/scalar_subquery.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,68 @@ impl ExecutionPlan for ScalarSubqueryExec {
254254
fn cardinality_effect(&self) -> CardinalityEffect {
255255
CardinalityEffect::Equal
256256
}
257+
258+
#[cfg(feature = "proto")]
259+
fn try_to_proto(
260+
&self,
261+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
262+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
263+
use datafusion_proto_models::protobuf;
264+
265+
let input = ctx.encode_child(self.input())?;
266+
// Subquery indices are positional and recovered during decoding.
267+
let subqueries =
268+
ctx.encode_children(self.subqueries().iter().map(|subquery| &subquery.plan))?;
269+
Ok(Some(protobuf::PhysicalPlanNode {
270+
physical_plan_type: Some(
271+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new(
272+
protobuf::ScalarSubqueryExecNode {
273+
input: Some(Box::new(input)),
274+
subqueries,
275+
},
276+
)),
277+
),
278+
}))
279+
}
280+
}
281+
282+
#[cfg(feature = "proto")]
283+
impl ScalarSubqueryExec {
284+
/// Reconstruct a [`ScalarSubqueryExec`] from its protobuf representation.
285+
pub fn try_from_proto(
286+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
287+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
288+
) -> Result<Arc<dyn ExecutionPlan>> {
289+
use datafusion_proto_models::protobuf;
290+
291+
let scalar_subquery = crate::expect_plan_variant!(
292+
node,
293+
protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery,
294+
"ScalarSubqueryExec",
295+
);
296+
let results = ScalarSubqueryResults::new(scalar_subquery.subqueries.len());
297+
let input_node = scalar_subquery.input.as_deref().ok_or_else(|| {
298+
datafusion_common::internal_datafusion_err!(
299+
"ScalarSubqueryExec is missing required field 'input'"
300+
)
301+
})?;
302+
// The input's ScalarSubqueryExpr nodes must share this results container.
303+
let input =
304+
ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?;
305+
let subqueries = scalar_subquery
306+
.subqueries
307+
.iter()
308+
.enumerate()
309+
.map(|(index, plan)| {
310+
Ok(ScalarSubqueryLink {
311+
plan: ctx.decode_child(plan)?,
312+
index: SubqueryIndex::new(index),
313+
})
314+
})
315+
.collect::<Result<Vec<_>>>()?;
316+
317+
Ok(Arc::new(Self::new(input, subqueries, results)))
318+
}
257319
}
258320

259321
/// Wait for the subquery execution future to complete.

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 38 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ use datafusion_datasource_parquet::source::ParquetSource;
4949
#[cfg(feature = "parquet")]
5050
use datafusion_execution::object_store::ObjectStoreUrl;
5151
use datafusion_execution::{FunctionRegistry, TaskContext};
52-
use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};
52+
use datafusion_expr::physical_planning_context::ScalarSubqueryResults;
5353
use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
5454
use datafusion_functions_table::generate_series::{
5555
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
@@ -85,7 +85,7 @@ use datafusion_physical_plan::proto::{
8585
ExecutionPlanEncodeCtx,
8686
};
8787
use datafusion_physical_plan::repartition::RepartitionExec;
88-
use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};
88+
use datafusion_physical_plan::scalar_subquery::ScalarSubqueryExec;
8989
use datafusion_physical_plan::sorts::sort::SortExec;
9090
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
9191
use datafusion_physical_plan::union::{InterleaveExec, UnionExec};
@@ -811,8 +811,8 @@ pub trait PhysicalPlanNodeExt: Sized {
811811
PhysicalPlanType::Buffer(_) => {
812812
BufferExec::try_from_proto(self.node(), &decode_ctx)
813813
}
814-
PhysicalPlanType::ScalarSubquery(sq) => {
815-
self.try_into_scalar_subquery_physical_plan(sq, ctx, proto_converter)
814+
PhysicalPlanType::ScalarSubquery(_) => {
815+
ScalarSubqueryExec::try_from_proto(self.node(), &decode_ctx)
816816
}
817817
}
818818
}
@@ -872,14 +872,6 @@ pub trait PhysicalPlanNodeExt: Sized {
872872
return Ok(node);
873873
}
874874

875-
if let Some(exec) = plan.downcast_ref::<ScalarSubqueryExec>() {
876-
return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec(
877-
exec,
878-
codec,
879-
proto_converter,
880-
);
881-
}
882-
883875
let mut buf: Vec<u8> = vec![];
884876
match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {
885877
Ok(_) => {
@@ -1950,38 +1942,27 @@ pub trait PhysicalPlanNodeExt: Sized {
19501942
BufferExec::try_from_proto(&node, &decode_ctx)
19511943
}
19521944

1945+
#[deprecated(
1946+
since = "55.0.0",
1947+
note = "unused by DataFusion; `ScalarSubqueryExec` deserializes itself via `ScalarSubqueryExec::try_from_proto`"
1948+
)]
19531949
fn try_into_scalar_subquery_physical_plan(
19541950
&self,
19551951
sq: &protobuf::ScalarSubqueryExecNode,
19561952
ctx: &PhysicalPlanDecodeContext<'_>,
19571953
proto_converter: &dyn PhysicalProtoConverterExtension,
19581954
) -> Result<Arc<dyn ExecutionPlan>> {
1959-
// First, deserialize the main input plan. We set up the subquery results
1960-
// container first, so that ScalarSubqueryExpr nodes can reference it.
1961-
let subquery_results = ScalarSubqueryResults::new(sq.subqueries.len());
1962-
let input_ctx = ctx.with_scalar_subquery_results(subquery_results.clone());
1963-
let input = into_physical_plan(&sq.input, &input_ctx, proto_converter)?;
1964-
1965-
// Now deserialize the subquery children.
1966-
let subqueries: Vec<ScalarSubqueryLink> = sq
1967-
.subqueries
1968-
.iter()
1969-
.enumerate()
1970-
.map(|(index, sq_plan)| {
1971-
let plan =
1972-
sq_plan.try_into_physical_plan_with_context(ctx, proto_converter)?;
1973-
Ok(ScalarSubqueryLink {
1974-
plan,
1975-
index: SubqueryIndex::new(index),
1976-
})
1977-
})
1978-
.collect::<Result<Vec<_>>>()?;
1979-
1980-
Ok(Arc::new(ScalarSubqueryExec::new(
1981-
input,
1982-
subqueries,
1983-
subquery_results,
1984-
)))
1955+
let node = protobuf::PhysicalPlanNode {
1956+
physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new(
1957+
sq.clone(),
1958+
))),
1959+
};
1960+
let decoder = ConverterPlanDecoder {
1961+
ctx,
1962+
proto_converter,
1963+
};
1964+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
1965+
ScalarSubqueryExec::try_from_proto(&node, &decode_ctx)
19851966
}
19861967

19871968
#[deprecated(
@@ -2866,35 +2847,22 @@ pub trait PhysicalPlanNodeExt: Sized {
28662847
.ok_or_else(|| internal_datafusion_err!("BufferExec is not serializable"))
28672848
}
28682849

2850+
#[deprecated(
2851+
since = "55.0.0",
2852+
note = "unused by DataFusion; `ScalarSubqueryExec` serializes itself via `ExecutionPlan::try_to_proto`"
2853+
)]
28692854
fn try_from_scalar_subquery_exec(
28702855
exec: &ScalarSubqueryExec,
28712856
codec: &dyn PhysicalExtensionCodec,
28722857
proto_converter: &dyn PhysicalProtoConverterExtension,
28732858
) -> Result<protobuf::PhysicalPlanNode> {
2874-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2875-
Arc::clone(exec.input()),
2859+
let encoder = ConverterPlanEncoder {
28762860
codec,
28772861
proto_converter,
2878-
)?;
2879-
let subqueries = exec
2880-
.subqueries()
2881-
.iter()
2882-
.map(|sq| {
2883-
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
2884-
Arc::clone(&sq.plan),
2885-
codec,
2886-
proto_converter,
2887-
)
2888-
})
2889-
.collect::<Result<Vec<_>>>()?;
2890-
2891-
Ok(protobuf::PhysicalPlanNode {
2892-
physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new(
2893-
protobuf::ScalarSubqueryExecNode {
2894-
input: Some(Box::new(input)),
2895-
subqueries,
2896-
},
2897-
))),
2862+
};
2863+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
2864+
exec.try_to_proto(&encode_ctx)?.ok_or_else(|| {
2865+
internal_datafusion_err!("ScalarSubqueryExec is not serializable")
28982866
})
28992867
}
29002868
}
@@ -3485,6 +3453,16 @@ impl ExecutionPlanDecode for ConverterPlanDecoder<'_, '_> {
34853453
self.proto_converter.proto_to_execution_plan(node, self.ctx)
34863454
}
34873455

3456+
fn decode_plan_with_scalar_subquery_results(
3457+
&self,
3458+
node: &protobuf::PhysicalPlanNode,
3459+
results: ScalarSubqueryResults,
3460+
) -> Result<Arc<dyn ExecutionPlan>> {
3461+
let scoped_ctx = self.ctx.with_scalar_subquery_results(results);
3462+
self.proto_converter
3463+
.proto_to_execution_plan(node, &scoped_ctx)
3464+
}
3465+
34883466
fn decode_expr(
34893467
&self,
34903468
node: &protobuf::PhysicalExprNode,

0 commit comments

Comments
 (0)