Skip to content

Commit ffdf4e6

Browse files
mattp5657Matthew Patton
authored andcommitted
refactor(proto): migrate AsyncFuncExec to self-serializing proto (apache#23825)
Closes apache#23514. Part of apache#23494. Migrate `AsyncFuncExec` proto encode/decode into the plan itself via `try_to_proto` / `try_from_proto`, removing its central-arm handling in `physical_plan/mod.rs`. ## Rationale `datafusion-proto` currently downcasts `AsyncFuncExec` in the central encode match and rebuilds it inline on decode. apache#23495 introduced self-serializing hooks so each plan owns its own wire format. This moves `AsyncFuncExec` onto that pattern. ## Changes - Added `try_to_proto` / `AsyncFuncExec::try_from_proto`, wired into the decode dispatch - Removed the central encode downcast branch - Old helper methods kept as `#[deprecated]` stubs, per existing convention - Wire format unchanged ## Testing Existing `roundtrip_async_func_exec` integration test now exercises the new hooks (old path deleted, so it's the only path left). `cargo fmt` + `cargo clippy --all-targets --features proto -- -D warnings` clean. ## User-facing changes No Co-authored-by: Matthew Patton <[email protected]>
1 parent 8dbb171 commit ffdf4e6

2 files changed

Lines changed: 105 additions & 64 deletions

File tree

‎datafusion/physical-plan/src/async_func.rs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,83 @@ impl ExecutionPlan for AsyncFuncExec {
246246
fn metrics(&self) -> Option<MetricsSet> {
247247
Some(self.metrics.clone_inner())
248248
}
249+
250+
#[cfg(feature = "proto")]
251+
fn try_to_proto(
252+
&self,
253+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
254+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
255+
use datafusion_proto_models::protobuf;
256+
let input = ctx.encode_child(self.input())?;
257+
let async_exprs =
258+
ctx.encode_expressions(self.async_exprs.iter().map(|e| &e.func))?;
259+
let async_expr_names = self
260+
.async_exprs
261+
.iter()
262+
.map(|e| e.name().to_string())
263+
.collect();
264+
Ok(Some(protobuf::PhysicalPlanNode {
265+
physical_plan_type: Some(
266+
protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc(Box::new(
267+
protobuf::AsyncFuncExecNode {
268+
input: Some(Box::new(input)),
269+
async_exprs,
270+
async_expr_names,
271+
},
272+
)),
273+
),
274+
}))
275+
}
276+
}
277+
278+
#[cfg(feature = "proto")]
279+
impl AsyncFuncExec {
280+
/// Reconstruct an [`AsyncFuncExec`] from its protobuf representation.
281+
///
282+
/// The exact inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
283+
/// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one
284+
/// signature. Child plans and expressions are decoded recursively via the
285+
/// [`ExecutionPlanDecodeCtx`].
286+
///
287+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
288+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
289+
/// [`ExecutionPlanDecodeCtx`]: crate::proto::ExecutionPlanDecodeCtx
290+
pub fn try_from_proto(
291+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
292+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
293+
) -> Result<Arc<dyn ExecutionPlan>> {
294+
use datafusion_proto_models::protobuf;
295+
let async_func = crate::expect_plan_variant!(
296+
node,
297+
protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc,
298+
"AsyncFuncExec",
299+
);
300+
let input = ctx.decode_required_child(
301+
async_func.input.as_deref(),
302+
"AsyncFuncExec",
303+
"input",
304+
)?;
305+
let input_schema = input.schema();
306+
assert_eq_or_internal_err!(
307+
async_func.async_exprs.len(),
308+
async_func.async_expr_names.len(),
309+
"AsyncFuncExecNode async_exprs length does not match async_expr_names"
310+
);
311+
let async_exprs = async_func
312+
.async_exprs
313+
.iter()
314+
.zip(async_func.async_expr_names.iter())
315+
.map(|(expr, name)| {
316+
let physical_expr = ctx.decode_expr(expr, input_schema.as_ref())?;
317+
Ok(Arc::new(AsyncFuncExpr::try_new(
318+
name.clone(),
319+
physical_expr,
320+
input_schema.as_ref(),
321+
)?))
322+
})
323+
.collect::<Result<Vec<_>>>()?;
324+
Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?))
325+
}
249326
}
250327

251328
struct CoalesceInputStream {

‎datafusion/proto/src/physical_plan/mod.rs‎

Lines changed: 28 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ use datafusion_functions_table::generate_series::{
5757
Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue,
5858
};
5959
use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
60-
use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;
6160
use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
6261
use datafusion_physical_expr::{LexOrdering, LexRequirement};
6362
use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx;
@@ -855,8 +854,8 @@ pub trait PhysicalPlanNodeExt: Sized {
855854
PhysicalPlanType::SortMergeJoin(_) => {
856855
SortMergeJoinExec::try_from_proto(self.node(), &decode_ctx)
857856
}
858-
PhysicalPlanType::AsyncFunc(async_func) => {
859-
self.try_into_async_func_physical_plan(async_func, ctx, proto_converter)
857+
PhysicalPlanType::AsyncFunc(_) => {
858+
AsyncFuncExec::try_from_proto(self.node(), &decode_ctx)
860859
}
861860
PhysicalPlanType::Buffer(_) => {
862861
BufferExec::try_from_proto(self.node(), &decode_ctx)
@@ -958,14 +957,6 @@ pub trait PhysicalPlanNodeExt: Sized {
958957
return Ok(node);
959958
}
960959

961-
if let Some(exec) = plan.downcast_ref::<AsyncFuncExec>() {
962-
return protobuf::PhysicalPlanNode::try_from_async_func_exec(
963-
exec,
964-
codec,
965-
proto_converter,
966-
);
967-
}
968-
969960
if let Some(exec) = plan.downcast_ref::<ScalarSubqueryExec>() {
970961
return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec(
971962
exec,
@@ -2244,41 +2235,27 @@ pub trait PhysicalPlanNodeExt: Sized {
22442235
CooperativeExec::try_from_proto(&node, &decode_ctx)
22452236
}
22462237

2238+
#[deprecated(
2239+
since = "55.0.0",
2240+
note = "unused by DataFusion; `AsyncFuncExec` deserializes itself via `AsyncFuncExec::try_from_proto`"
2241+
)]
22472242
fn try_into_async_func_physical_plan(
22482243
&self,
22492244
async_func: &protobuf::AsyncFuncExecNode,
22502245
ctx: &PhysicalPlanDecodeContext<'_>,
22512246
proto_converter: &dyn PhysicalProtoConverterExtension,
22522247
) -> Result<Arc<dyn ExecutionPlan>> {
2253-
let input: Arc<dyn ExecutionPlan> =
2254-
into_physical_plan(&async_func.input, ctx, proto_converter)?;
2255-
2256-
if async_func.async_exprs.len() != async_func.async_expr_names.len() {
2257-
return internal_err!(
2258-
"AsyncFuncExecNode async_exprs length does not match async_expr_names"
2259-
);
2260-
}
2261-
2262-
let async_exprs = async_func
2263-
.async_exprs
2264-
.iter()
2265-
.zip(async_func.async_expr_names.iter())
2266-
.map(|(expr, name)| {
2267-
let physical_expr = proto_converter.proto_to_physical_expr(
2268-
expr,
2269-
input.schema().as_ref(),
2270-
ctx,
2271-
)?;
2272-
2273-
Ok(Arc::new(AsyncFuncExpr::try_new(
2274-
name.clone(),
2275-
physical_expr,
2276-
input.schema().as_ref(),
2277-
)?))
2278-
})
2279-
.collect::<Result<Vec<_>>>()?;
2280-
2281-
Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?))
2248+
let node = protobuf::PhysicalPlanNode {
2249+
physical_plan_type: Some(PhysicalPlanType::AsyncFunc(Box::new(
2250+
async_func.clone(),
2251+
))),
2252+
};
2253+
let decoder = ConverterPlanDecoder {
2254+
ctx,
2255+
proto_converter,
2256+
};
2257+
let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder);
2258+
AsyncFuncExec::try_from_proto(&node, &decode_ctx)
22822259
}
22832260

22842261
#[deprecated(
@@ -3333,35 +3310,22 @@ pub trait PhysicalPlanNodeExt: Sized {
33333310
Ok(None)
33343311
}
33353312

3313+
#[deprecated(
3314+
since = "55.0.0",
3315+
note = "unused by DataFusion; `AsyncFuncExec` serializes itself via `ExecutionPlan::try_to_proto`"
3316+
)]
33363317
fn try_from_async_func_exec(
33373318
exec: &AsyncFuncExec,
3338-
codec: &dyn PhysicalExtensionCodec,
3319+
extension_codec: &dyn PhysicalExtensionCodec,
33393320
proto_converter: &dyn PhysicalProtoConverterExtension,
33403321
) -> Result<protobuf::PhysicalPlanNode> {
3341-
let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
3342-
Arc::clone(exec.input()),
3343-
codec,
3322+
let encoder = ConverterPlanEncoder {
3323+
codec: extension_codec,
33443324
proto_converter,
3345-
)?;
3346-
3347-
let mut async_exprs = vec![];
3348-
let mut async_expr_names = vec![];
3349-
3350-
for async_expr in exec.async_exprs() {
3351-
async_exprs
3352-
.push(proto_converter.physical_expr_to_proto(&async_expr.func, codec)?);
3353-
async_expr_names.push(async_expr.name.clone())
3354-
}
3355-
3356-
Ok(protobuf::PhysicalPlanNode {
3357-
physical_plan_type: Some(PhysicalPlanType::AsyncFunc(Box::new(
3358-
protobuf::AsyncFuncExecNode {
3359-
input: Some(Box::new(input)),
3360-
async_exprs,
3361-
async_expr_names,
3362-
},
3363-
))),
3364-
})
3325+
};
3326+
let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder);
3327+
exec.try_to_proto(&encode_ctx)?
3328+
.ok_or_else(|| internal_datafusion_err!("AsyncFuncExec is not serializable"))
33653329
}
33663330

33673331
#[deprecated(

0 commit comments

Comments
 (0)