Skip to content

Commit 4c4f169

Browse files
authored
refactor(proto): migrate SortExec and SortPreservingMergeExec serde (apache#23794)
## Which issue does this PR close? - Closes apache#23505. ## Rationale for this change Part of epic apache#23494. Moves `SortExec` and `SortPreservingMergeExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add protobuf serialization and deserialization to the physical plan implementations and deprecate the corresponding central proto methods. The wire format remains unchanged. ## Are these changes tested? Yes, existing round-trip tests cover these plans. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change.
1 parent 60cdf8a commit 4c4f169

3 files changed

Lines changed: 256 additions & 196 deletions

File tree

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1553,6 +1553,120 @@ impl ExecutionPlan for SortExec {
15531553
updated_node: Some(new_sort),
15541554
})
15551555
}
1556+
#[cfg(feature = "proto")]
1557+
fn try_to_proto(
1558+
&self,
1559+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
1560+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
1561+
use datafusion_proto_models::protobuf;
1562+
let input = ctx.encode_child(self.input())?;
1563+
let expr = self
1564+
.expr()
1565+
.iter()
1566+
.map(|sort_expr| {
1567+
let sort_node = Box::new(protobuf::PhysicalSortExprNode {
1568+
expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)),
1569+
asc: !sort_expr.options.descending,
1570+
nulls_first: sort_expr.options.nulls_first,
1571+
});
1572+
Ok(protobuf::PhysicalExprNode {
1573+
expr_id: None,
1574+
expr_type: Some(protobuf::physical_expr_node::ExprType::Sort(
1575+
sort_node,
1576+
)),
1577+
})
1578+
})
1579+
.collect::<Result<Vec<_>>>()?;
1580+
let dynamic_filter = match self.dynamic_filter_expr() {
1581+
Some(df) => {
1582+
let df_expr: Arc<dyn PhysicalExpr> = df;
1583+
Some(ctx.encode_expr(&df_expr)?)
1584+
}
1585+
None => None,
1586+
};
1587+
Ok(Some(protobuf::PhysicalPlanNode {
1588+
physical_plan_type: Some(
1589+
protobuf::physical_plan_node::PhysicalPlanType::Sort(Box::new(
1590+
protobuf::SortExecNode {
1591+
input: Some(Box::new(input)),
1592+
expr,
1593+
fetch: match self.fetch() {
1594+
Some(n) => n as i64,
1595+
None => -1,
1596+
},
1597+
preserve_partitioning: self.preserve_partitioning(),
1598+
dynamic_filter,
1599+
},
1600+
)),
1601+
),
1602+
}))
1603+
}
1604+
}
1605+
1606+
#[cfg(feature = "proto")]
1607+
impl SortExec {
1608+
pub fn try_from_proto(
1609+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
1610+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
1611+
) -> Result<Arc<dyn ExecutionPlan>> {
1612+
use datafusion_proto_models::protobuf;
1613+
use protobuf::physical_expr_node::ExprType;
1614+
let sort = crate::expect_plan_variant!(
1615+
node,
1616+
protobuf::physical_plan_node::PhysicalPlanType::Sort,
1617+
"SortExec",
1618+
);
1619+
let input =
1620+
ctx.decode_required_child(sort.input.as_deref(), "SortExec", "input")?;
1621+
let input_schema = input.schema();
1622+
let exprs = sort
1623+
.expr
1624+
.iter()
1625+
.map(|expr| {
1626+
let Some(ExprType::Sort(sort_expr)) = expr.expr_type.as_ref() else {
1627+
return datafusion_common::internal_err!(
1628+
"SortExec expr must be a sort expression"
1629+
);
1630+
};
1631+
let expr_node = sort_expr.expr.as_deref().ok_or_else(|| {
1632+
internal_datafusion_err!(
1633+
"SortExec sort expression is missing its inner expr"
1634+
)
1635+
})?;
1636+
Ok(PhysicalSortExpr {
1637+
expr: ctx.decode_expr(expr_node, input_schema.as_ref())?,
1638+
options: arrow::compute::SortOptions {
1639+
descending: !sort_expr.asc,
1640+
nulls_first: sort_expr.nulls_first,
1641+
},
1642+
})
1643+
})
1644+
.collect::<Result<Vec<_>>>()?;
1645+
let Some(ordering) = LexOrdering::new(exprs) else {
1646+
return datafusion_common::internal_err!("SortExec requires an ordering");
1647+
};
1648+
let fetch = (sort.fetch >= 0).then_some(sort.fetch as usize);
1649+
let new_sort = SortExec::new(ordering, input)
1650+
.with_fetch(fetch)
1651+
.with_preserve_partitioning(sort.preserve_partitioning);
1652+
1653+
let new_sort = if let Some(df_proto) = &sort.dynamic_filter {
1654+
let df_expr =
1655+
ctx.decode_expr(df_proto, new_sort.input().schema().as_ref())?;
1656+
let df = (df_expr as Arc<dyn std::any::Any + Send + Sync>)
1657+
.downcast::<DynamicFilterPhysicalExpr>()
1658+
.map_err(|_| {
1659+
internal_datafusion_err!(
1660+
"SortExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"
1661+
)
1662+
})?;
1663+
new_sort.with_dynamic_filter_expr(df)?
1664+
} else {
1665+
new_sort
1666+
};
1667+
1668+
Ok(Arc::new(new_sort))
1669+
}
15561670
}
15571671

15581672
#[cfg(test)]

‎datafusion/physical-plan/src/sorts/sort_preserving_merge.rs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,98 @@ impl ExecutionPlan for SortPreservingMergeExec {
437437
.with_fetch(self.fetch()),
438438
)))
439439
}
440+
#[cfg(feature = "proto")]
441+
fn try_to_proto(
442+
&self,
443+
ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>,
444+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
445+
use datafusion_proto_models::protobuf;
446+
let input = ctx.encode_child(self.input())?;
447+
let expr = self
448+
.expr()
449+
.iter()
450+
.map(|e| {
451+
Ok(protobuf::PhysicalExprNode {
452+
expr_id: None,
453+
expr_type: Some(protobuf::physical_expr_node::ExprType::Sort(
454+
Box::new(protobuf::PhysicalSortExprNode {
455+
expr: Some(Box::new(ctx.encode_expr(&e.expr)?)),
456+
asc: !e.options.descending,
457+
nulls_first: e.options.nulls_first,
458+
}),
459+
)),
460+
})
461+
})
462+
.collect::<Result<Vec<_>>>()?;
463+
Ok(Some(protobuf::PhysicalPlanNode {
464+
physical_plan_type: Some(
465+
protobuf::physical_plan_node::PhysicalPlanType::SortPreservingMerge(
466+
Box::new(protobuf::SortPreservingMergeExecNode {
467+
input: Some(Box::new(input)),
468+
expr,
469+
fetch: self.fetch().map(|f| f as i64).unwrap_or(-1),
470+
}),
471+
),
472+
),
473+
}))
474+
}
475+
}
476+
477+
#[cfg(feature = "proto")]
478+
impl SortPreservingMergeExec {
479+
pub fn try_from_proto(
480+
node: &datafusion_proto_models::protobuf::PhysicalPlanNode,
481+
ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,
482+
) -> Result<Arc<dyn ExecutionPlan>> {
483+
use arrow::compute::SortOptions;
484+
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
485+
use datafusion_proto_models::protobuf;
486+
let spm = crate::expect_plan_variant!(
487+
node,
488+
protobuf::physical_plan_node::PhysicalPlanType::SortPreservingMerge,
489+
"SortPreservingMergeExec",
490+
);
491+
let input = ctx.decode_required_child(
492+
spm.input.as_deref(),
493+
"SortPreservingMergeExec",
494+
"input",
495+
)?;
496+
let input_schema = input.schema();
497+
let exprs = spm
498+
.expr
499+
.iter()
500+
.map(|e| {
501+
let sort = match &e.expr_type {
502+
Some(protobuf::physical_expr_node::ExprType::Sort(s)) => s,
503+
_ => {
504+
return internal_err!(
505+
"SortPreservingMergeExec expression is not a sort expression"
506+
);
507+
}
508+
};
509+
let expr = ctx.decode_required_expr(
510+
sort.expr.as_deref(),
511+
input_schema.as_ref(),
512+
"SortPreservingMergeExec",
513+
"sort expression",
514+
)?;
515+
Ok(PhysicalSortExpr {
516+
expr,
517+
options: SortOptions {
518+
descending: !sort.asc,
519+
nulls_first: sort.nulls_first,
520+
},
521+
})
522+
})
523+
.collect::<Result<Vec<_>>>()?;
524+
let Some(ordering) = LexOrdering::new(exprs) else {
525+
return internal_err!("SortPreservingMergeExec requires an ordering");
526+
};
527+
let fetch = (spm.fetch >= 0).then_some(spm.fetch as usize);
528+
Ok(Arc::new(
529+
SortPreservingMergeExec::new(ordering, input).with_fetch(fetch),
530+
))
531+
}
440532
}
441533

442534
#[cfg(test)]

0 commit comments

Comments
 (0)