Skip to content

Commit e225c8b

Browse files
adriangbclaude
andcommitted
prototype: migrate 7 self-contained ExecutionPlans onto the proto hook (apache#22419)
Fleet-migrated Filter, Empty, CoalesceBatches, Union/Interleave, Sort, Repartition, and HashJoin onto try_to_proto/try_from_proto. Central encode downcast arms deleted; decode arms reduced to one-liners. All 190 proto_integration tests pass. Findings proven by the prototype: - Partitioning and PhysicalSortExpr wrapping (serialize_partitioning / serialize_physical_sort_exprs) inline cleanly from datafusion-proto-models; no datafusion-proto-only helper needed. - Schema and ScalarValue conversions come from datafusion-proto-common (already a physical-plan dep under the proto feature). - Enum conversions (JoinType/NullEquality/PartitionMode/JoinSide) must be inlined as by-NAME matches; proto vs common variant ordering differs. - The hook must follow downcast_delegate() before dispatch, else wrapper plans serialize wrong (caught by serialize_uses_downcast_delegate). - No migrated plan needed the PhysicalExtensionCodec or function registry. Added ExecutionPlanDecodeCtx::decode_children for the N-inputs shape. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01SfBPC2yYCwHUn4V8pQCVj3
1 parent 6991675 commit e225c8b

9 files changed

Lines changed: 952 additions & 82 deletions

File tree

‎datafusion/physical-plan/src/coalesce_batches.rs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,61 @@ impl ExecutionPlan for CoalesceBatchesExec {
290290
) as Arc<dyn ExecutionPlan>)
291291
})
292292
}
293+
294+
#[cfg(feature = "proto")]
295+
fn try_to_proto(
296+
&self,
297+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
298+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
299+
use datafusion_proto_models::protobuf;
300+
let input = ctx.encode_child(self.input())?;
301+
Ok(Some(protobuf::PhysicalPlanNode {
302+
physical_plan_type: Some(
303+
protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches(
304+
Box::new(protobuf::CoalesceBatchesExecNode {
305+
input: Some(Box::new(input)),
306+
target_batch_size: self.target_batch_size() as u32,
307+
fetch: self.fetch().map(|n| n as u32),
308+
}),
309+
),
310+
),
311+
}))
312+
}
313+
}
314+
315+
#[cfg(feature = "proto")]
316+
#[expect(deprecated)]
317+
impl CoalesceBatchesExec {
318+
/// Reconstruct a [`CoalesceBatchesExec`] from its protobuf representation.
319+
///
320+
/// The exact inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
321+
/// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one
322+
/// signature. The child plan is decoded recursively via the
323+
/// [`ExecutionPlanDecodeCtx`].
324+
///
325+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
326+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
327+
/// [`ExecutionPlanDecodeCtx`]: crate::proto::ExecutionPlanDecodeCtx
328+
pub fn try_from_proto(
329+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
330+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
331+
) -> Result<Arc<dyn ExecutionPlan>> {
332+
use datafusion_proto_models::protobuf;
333+
let coalesce_batches = crate::expect_plan_variant!(
334+
node,
335+
protobuf::physical_plan_node::PhysicalPlanType::CoalesceBatches,
336+
"CoalesceBatchesExec",
337+
);
338+
let input = ctx.decode_required_child(
339+
coalesce_batches.input.as_deref(),
340+
"CoalesceBatchesExec",
341+
"input",
342+
)?;
343+
Ok(Arc::new(
344+
CoalesceBatchesExec::new(input, coalesce_batches.target_batch_size as usize)
345+
.with_fetch(coalesce_batches.fetch.map(|f| f as usize)),
346+
))
347+
}
293348
}
294349

295350
/// Stream for [`CoalesceBatchesExec`]. See [`CoalesceBatchesExec`] for more details.

‎datafusion/physical-plan/src/empty.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,55 @@ impl ExecutionPlan for EmptyExec {
185185

186186
Ok(Arc::new(stats))
187187
}
188+
189+
#[cfg(feature = "proto")]
190+
fn try_to_proto(
191+
&self,
192+
_ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
193+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
194+
use datafusion_proto_models::protobuf;
195+
// EmptyExec is a leaf plan carrying only its schema on the wire.
196+
let schema = self.schema().as_ref().try_into()?;
197+
Ok(Some(protobuf::PhysicalPlanNode {
198+
physical_plan_type: Some(
199+
protobuf::physical_plan_node::PhysicalPlanType::Empty(
200+
protobuf::EmptyExecNode {
201+
schema: Some(schema),
202+
},
203+
),
204+
),
205+
}))
206+
}
207+
}
208+
209+
#[cfg(feature = "proto")]
210+
impl EmptyExec {
211+
/// Reconstruct an [`EmptyExec`] from its protobuf representation.
212+
///
213+
/// The inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
214+
/// [`PhysicalPlanNode`] and reads back the schema (this leaf plan has no
215+
/// children or expressions to decode).
216+
///
217+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
218+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
219+
pub fn try_from_proto(
220+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
221+
_ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
222+
) -> Result<Arc<dyn ExecutionPlan>> {
223+
use datafusion_proto_models::protobuf;
224+
let empty = crate::expect_plan_variant!(
225+
node,
226+
protobuf::physical_plan_node::PhysicalPlanType::Empty,
227+
"EmptyExec",
228+
);
229+
let schema = empty.schema.as_ref().ok_or_else(|| {
230+
datafusion_common::internal_datafusion_err!(
231+
"EmptyExec is missing required field 'schema'"
232+
)
233+
})?;
234+
let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?);
235+
Ok(Arc::new(EmptyExec::new(schema)))
236+
}
188237
}
189238

190239
#[cfg(test)]

‎datafusion/physical-plan/src/filter.rs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -812,6 +812,102 @@ impl ExecutionPlan for FilterExec {
812812
.ok()
813813
})
814814
}
815+
816+
#[cfg(feature = "proto")]
817+
fn try_to_proto(
818+
&self,
819+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
820+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
821+
use datafusion_proto_models::protobuf;
822+
let input = ctx.encode_child(self.input())?;
823+
let expr = ctx.encode_expr(self.predicate())?;
824+
// Preserve the exact wire format: `None` (full projection) is serialized
825+
// as the identity projection `[0, 1, .., num_fields - 1]` so that it is
826+
// distinguishable from an explicit projection on decode.
827+
let projection = match self.projection() {
828+
None => (0..self.input().schema().fields().len())
829+
.map(|i| i as u32)
830+
.collect(),
831+
Some(v) => v.iter().map(|x| *x as u32).collect(),
832+
};
833+
Ok(Some(protobuf::PhysicalPlanNode {
834+
physical_plan_type: Some(
835+
protobuf::physical_plan_node::PhysicalPlanType::Filter(Box::new(
836+
protobuf::FilterExecNode {
837+
input: Some(Box::new(input)),
838+
expr: Some(expr),
839+
default_filter_selectivity: self.default_selectivity() as u32,
840+
projection,
841+
batch_size: self.batch_size() as u32,
842+
fetch: self.fetch().map(|f| f as u32),
843+
},
844+
)),
845+
),
846+
}))
847+
}
848+
}
849+
850+
#[cfg(feature = "proto")]
851+
impl FilterExec {
852+
/// Reconstruct a [`FilterExec`] from its protobuf representation.
853+
///
854+
/// The exact inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole
855+
/// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one
856+
/// signature.
857+
///
858+
/// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode
859+
/// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto
860+
pub fn try_from_proto(
861+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
862+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
863+
) -> Result<Arc<dyn ExecutionPlan>> {
864+
use datafusion_proto_models::protobuf;
865+
let filter = crate::expect_plan_variant!(
866+
node,
867+
protobuf::physical_plan_node::PhysicalPlanType::Filter,
868+
"FilterExec",
869+
);
870+
let input =
871+
ctx.decode_required_child(filter.input.as_deref(), "FilterExec", "input")?;
872+
let predicate = ctx.decode_required_expr(
873+
filter.expr.as_ref(),
874+
input.schema().as_ref(),
875+
"FilterExec",
876+
"expr",
877+
)?;
878+
879+
let filter_selectivity = filter.default_filter_selectivity.try_into();
880+
// Preserve the `None` state across proto boundaries. Proto cannot distinguish
881+
// between `None` (full projection) and `Some(vec![])` (empty projection) since
882+
// both serialize as an empty list. If all columns are included, we reconstruct
883+
// `None` to avoid losing this semantic distinction on deserialization.
884+
let num_fields = input.schema().fields().len();
885+
let mut is_full_projection = filter.projection.len() == num_fields;
886+
let mut projection_vec: Vec<usize> = Vec::with_capacity(filter.projection.len());
887+
for (i, idx) in filter.projection.iter().enumerate() {
888+
let idx = *idx as usize;
889+
is_full_projection &= idx == i;
890+
projection_vec.push(idx);
891+
}
892+
let projection = if is_full_projection {
893+
None
894+
} else {
895+
Some(projection_vec)
896+
};
897+
let filter = FilterExecBuilder::new(predicate, input)
898+
.apply_projection(projection)?
899+
.with_batch_size(filter.batch_size as usize)
900+
.with_fetch(filter.fetch.map(|f| f as usize))
901+
.build()?;
902+
match filter_selectivity {
903+
Ok(filter_selectivity) => Ok(Arc::new(
904+
filter.with_default_selectivity(filter_selectivity)?,
905+
)),
906+
Err(_) => Err(datafusion_common::internal_datafusion_err!(
907+
"filter_selectivity in PhysicalPlanNode is invalid "
908+
)),
909+
}
910+
}
815911
}
816912

817913
impl EmbeddedProjection for FilterExec {

0 commit comments

Comments
 (0)