Skip to content

Commit 962f5ba

Browse files
adriangbclaude
andcommitted
prototype: migrate 14 more ExecutionPlans onto the proto hook incl. AggregateExec (apache#22419)
Bucket A: CrossJoin, NestedLoopJoin, SortMergeJoin, SymmetricHashJoin, CoalescePartitions, Cooperative, Buffer, GlobalLimit, LocalLimit, PlaceholderRow, Unnest, Explain, Analyze. Bucket B (function-carrying, proves the ctx function-codec design): AggregateExec — UDAF round-trips purely through ctx.encode_udaf/decode_udaf with zero proto/codec/registry types crossing the boundary. All join enum conversions inlined by-name (StreamPartitionMode, JoinType, NullEquality, JoinSide). StringifiedPlan and UnnestOptions conversions (which lived only in datafusion-proto) inlined from proto-common/models. All 189 roundtrip tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 6b6fc39 commit 962f5ba

14 files changed

Lines changed: 2033 additions & 138 deletions

File tree

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 421 additions & 0 deletions
Large diffs are not rendered by default.

‎datafusion/physical-plan/src/analyze.rs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,106 @@ impl ExecutionPlan for AnalyzeExec {
303303
futures::stream::once(output),
304304
)))
305305
}
306+
307+
#[cfg(feature = "proto")]
308+
fn try_to_proto(
309+
&self,
310+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
311+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
312+
use datafusion_proto_models::protobuf;
313+
let input = ctx.encode_child(self.input())?;
314+
let (has_metric_categories, metric_categories) = match self.metric_categories() {
315+
Some(cats) => (true, cats.iter().map(|c| c.to_string()).collect()),
316+
None => (false, vec![]),
317+
};
318+
// Inline by-NAME match: the wire field is an i32; map the common enum to
319+
// the proto enum by variant (never a numeric cast) so a byte-identical
320+
// discriminant is written and a future variant is a compile error.
321+
let format = match self.format() {
322+
ExplainFormat::Indent => protobuf::ExplainFormat::Indent,
323+
ExplainFormat::Tree => protobuf::ExplainFormat::Tree,
324+
ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson,
325+
ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz,
326+
} as i32;
327+
let schema = self.schema().as_ref().try_into()?;
328+
Ok(Some(protobuf::PhysicalPlanNode {
329+
physical_plan_type: Some(
330+
protobuf::physical_plan_node::PhysicalPlanType::Analyze(Box::new(
331+
protobuf::AnalyzeExecNode {
332+
verbose: self.verbose(),
333+
show_statistics: self.show_statistics(),
334+
input: Some(Box::new(input)),
335+
schema: Some(schema),
336+
has_metric_categories,
337+
metric_categories,
338+
format,
339+
},
340+
)),
341+
),
342+
}))
343+
}
344+
}
345+
346+
#[cfg(feature = "proto")]
347+
impl AnalyzeExec {
348+
/// Reconstruct an [`AnalyzeExec`] from its protobuf representation.
349+
///
350+
/// The inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
351+
/// [`PhysicalPlanNode`], decodes the single child recursively via the
352+
/// [`ExecutionPlanDecodeCtx`], and reads back the flags, metric categories
353+
/// and output format.
354+
///
355+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
356+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
357+
/// [`ExecutionPlanDecodeCtx`]: crate::proto::ExecutionPlanDecodeCtx
358+
pub fn try_from_proto(
359+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
360+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
361+
) -> Result<Arc<dyn ExecutionPlan>> {
362+
use datafusion_proto_models::protobuf;
363+
let analyze = crate::expect_plan_variant!(
364+
node,
365+
protobuf::physical_plan_node::PhysicalPlanType::Analyze,
366+
"AnalyzeExec",
367+
);
368+
let input =
369+
ctx.decode_required_child(analyze.input.as_deref(), "AnalyzeExec", "input")?;
370+
let metric_categories = if analyze.has_metric_categories {
371+
let cats: Result<Vec<MetricCategory>> = analyze
372+
.metric_categories
373+
.iter()
374+
.map(|s| s.parse::<MetricCategory>())
375+
.collect();
376+
Some(cats?)
377+
} else {
378+
None
379+
};
380+
let pb_format =
381+
protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| {
382+
DataFusionError::Internal(format!(
383+
"Received an AnalyzeExecNode message with unknown ExplainFormat {}",
384+
analyze.format
385+
))
386+
})?;
387+
let format = match pb_format {
388+
protobuf::ExplainFormat::Indent => ExplainFormat::Indent,
389+
protobuf::ExplainFormat::Tree => ExplainFormat::Tree,
390+
protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON,
391+
protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz,
392+
};
393+
let schema = analyze.schema.as_ref().ok_or_else(|| {
394+
datafusion_common::internal_datafusion_err!(
395+
"AnalyzeExec is missing required field 'schema'"
396+
)
397+
})?;
398+
let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?);
399+
Ok(Arc::new(
400+
AnalyzeExec::builder(analyze.verbose, analyze.show_statistics, input, schema)
401+
.with_metric_categories(metric_categories)
402+
.with_format(format)
403+
.build(),
404+
))
405+
}
306406
}
307407

308408
/// Creates the output of AnalyzeExec as a RecordBatch

‎datafusion/physical-plan/src/buffer.rs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,48 @@ impl ExecutionPlan for BufferExec {
298298
Ok(Arc::new(Self::new(new_input, self.capacity)) as Arc<dyn ExecutionPlan>)
299299
})
300300
}
301+
302+
#[cfg(feature = "proto")]
303+
fn try_to_proto(
304+
&self,
305+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
306+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
307+
use datafusion_proto_models::protobuf;
308+
let input = ctx.encode_child(self.input())?;
309+
Ok(Some(protobuf::PhysicalPlanNode {
310+
physical_plan_type: Some(
311+
protobuf::physical_plan_node::PhysicalPlanType::Buffer(Box::new(
312+
protobuf::BufferExecNode {
313+
input: Some(Box::new(input)),
314+
capacity: self.capacity() as u64,
315+
},
316+
)),
317+
),
318+
}))
319+
}
320+
}
321+
322+
#[cfg(feature = "proto")]
323+
impl BufferExec {
324+
/// Reconstruct a [`BufferExec`] from its protobuf representation.
325+
///
326+
/// The exact inverse of [`ExecutionPlan::try_to_proto`].
327+
///
328+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
329+
pub fn try_from_proto(
330+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
331+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
332+
) -> Result<Arc<dyn ExecutionPlan>> {
333+
use datafusion_proto_models::protobuf;
334+
let buffer = crate::expect_plan_variant!(
335+
node,
336+
protobuf::physical_plan_node::PhysicalPlanType::Buffer,
337+
"BufferExec",
338+
);
339+
let input =
340+
ctx.decode_required_child(buffer.input.as_deref(), "BufferExec", "input")?;
341+
Ok(Arc::new(BufferExec::new(input, buffer.capacity as usize)))
342+
}
301343
}
302344

303345
/// Represents anything that occupies a capacity in a [MemoryBufferedStream].

‎datafusion/physical-plan/src/coalesce_partitions.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,56 @@ impl ExecutionPlan for CoalescePartitionsExec {
346346
}
347347
})
348348
}
349+
350+
#[cfg(feature = "proto")]
351+
fn try_to_proto(
352+
&self,
353+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
354+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
355+
use datafusion_proto_models::protobuf;
356+
let input = ctx.encode_child(self.input())?;
357+
Ok(Some(protobuf::PhysicalPlanNode {
358+
physical_plan_type: Some(
359+
protobuf::physical_plan_node::PhysicalPlanType::Merge(Box::new(
360+
protobuf::CoalescePartitionsExecNode {
361+
input: Some(Box::new(input)),
362+
fetch: self.fetch().map(|f| f as u32),
363+
},
364+
)),
365+
),
366+
}))
367+
}
368+
}
369+
370+
#[cfg(feature = "proto")]
371+
impl CoalescePartitionsExec {
372+
/// Reconstruct a [`CoalescePartitionsExec`] from its protobuf representation.
373+
///
374+
/// The exact inverse of [`ExecutionPlan::try_to_proto`]. Note the proto
375+
/// variant is named `Merge` (node [`CoalescePartitionsExecNode`]).
376+
///
377+
/// [`CoalescePartitionsExecNode`]: datafusion_proto_models::protobuf::CoalescePartitionsExecNode
378+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
379+
pub fn try_from_proto(
380+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
381+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
382+
) -> Result<Arc<dyn ExecutionPlan>> {
383+
use datafusion_proto_models::protobuf;
384+
let merge = crate::expect_plan_variant!(
385+
node,
386+
protobuf::physical_plan_node::PhysicalPlanType::Merge,
387+
"CoalescePartitionsExec",
388+
);
389+
let input = ctx.decode_required_child(
390+
merge.input.as_deref(),
391+
"CoalescePartitionsExec",
392+
"input",
393+
)?;
394+
Ok(Arc::new(
395+
CoalescePartitionsExec::new(input)
396+
.with_fetch(merge.fetch.map(|f| f as usize)),
397+
))
398+
}
349399
}
350400

351401
#[cfg(test)]

‎datafusion/physical-plan/src/coop.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,50 @@ impl ExecutionPlan for CooperativeExec {
369369
}
370370
}
371371
}
372+
373+
#[cfg(feature = "proto")]
374+
fn try_to_proto(
375+
&self,
376+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
377+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
378+
use datafusion_proto_models::protobuf;
379+
let input = ctx.encode_child(self.input())?;
380+
Ok(Some(protobuf::PhysicalPlanNode {
381+
physical_plan_type: Some(
382+
protobuf::physical_plan_node::PhysicalPlanType::Cooperative(Box::new(
383+
protobuf::CooperativeExecNode {
384+
input: Some(Box::new(input)),
385+
},
386+
)),
387+
),
388+
}))
389+
}
390+
}
391+
392+
#[cfg(feature = "proto")]
393+
impl CooperativeExec {
394+
/// Reconstruct a [`CooperativeExec`] from its protobuf representation.
395+
///
396+
/// The exact inverse of [`ExecutionPlan::try_to_proto`].
397+
///
398+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
399+
pub fn try_from_proto(
400+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
401+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
402+
) -> Result<Arc<dyn ExecutionPlan>> {
403+
use datafusion_proto_models::protobuf;
404+
let cooperative = crate::expect_plan_variant!(
405+
node,
406+
protobuf::physical_plan_node::PhysicalPlanType::Cooperative,
407+
"CooperativeExec",
408+
);
409+
let input = ctx.decode_required_child(
410+
cooperative.input.as_deref(),
411+
"CooperativeExec",
412+
"input",
413+
)?;
414+
Ok(Arc::new(CooperativeExec::new(input)))
415+
}
372416
}
373417

374418
/// Creates a [`CooperativeStream`] wrapper around the given [`RecordBatchStream`].

0 commit comments

Comments
 (0)