Skip to content

Commit 7f41edb

Browse files
committed
feat(proto): add DataSink serialization hook
DataSinkExec needs a generic way to delegate protobuf encoding to its wrapped sink. The new hook keeps sink-specific wire logic out of the central serializer while retaining the fallback for unmigrated sinks. Add focused coverage for child and sort-order encoding and verify that existing file sinks continue to use the compatibility path. Refs apache#23498
1 parent 39d5064 commit 7f41edb

3 files changed

Lines changed: 151 additions & 5 deletions

File tree

‎datafusion/datasource/Cargo.toml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ all-features = true
3434
backtrace = ["datafusion-common/backtrace"]
3535
compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"]
3636
default = ["compression"]
37-
# Enables the protobuf conversions for the file-scan leaf types owned by this
38-
# crate (`FileRange`, `PartitionedFile`, `FileGroup`). Off by default so
39-
# consumers that never serialize plans pay nothing.
40-
proto = ["dep:datafusion-proto-models"]
37+
# Enables protobuf conversions for datasource types and serialization hooks.
38+
# Off by default so consumers that never serialize plans pay nothing.
39+
proto = [
40+
"dep:datafusion-proto-models",
41+
"datafusion-physical-plan/proto",
42+
]
4143

4244
[dependencies]
4345
arrow = { workspace = true }

‎datafusion/datasource/src/sink.rs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,25 @@ pub trait DataSink: Any + DisplayAs + Debug + Send + Sync {
7171
data: SendableRecordBatchStream,
7272
context: &Arc<TaskContext>,
7373
) -> Result<u64>;
74+
75+
/// Serialize this sink into a full protobuf plan node, if it knows how.
76+
///
77+
/// [`DataSinkExec::try_to_proto`] encodes the shared child plan and required
78+
/// output ordering before delegating to this hook. Implementations only need
79+
/// to add their sink-specific fields.
80+
///
81+
/// Returning `Ok(None)` preserves the legacy central serialization fallback.
82+
#[cfg(feature = "proto")]
83+
fn try_to_proto(
84+
&self,
85+
_input: datafusion_proto_models::protobuf::PhysicalPlanNode,
86+
_sort_order: Option<
87+
datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
88+
>,
89+
_sink_schema: &Schema,
90+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
91+
Ok(None)
92+
}
7493
}
7594

7695
impl dyn DataSink {
@@ -268,6 +287,44 @@ impl ExecutionPlan for DataSinkExec {
268287
fn metrics(&self) -> Option<MetricsSet> {
269288
self.sink.metrics()
270289
}
290+
291+
/// Encodes the shared sink plan fields before delegating the sink-specific
292+
/// protobuf representation to [`DataSink::try_to_proto`].
293+
#[cfg(feature = "proto")]
294+
fn try_to_proto(
295+
&self,
296+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
297+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
298+
use datafusion_physical_expr::PhysicalSortExpr;
299+
use datafusion_proto_models::protobuf;
300+
301+
let input = ctx.encode_child(self.input())?;
302+
let sort_order = self
303+
.sort_order()
304+
.as_ref()
305+
.map(|requirements| {
306+
requirements
307+
.iter()
308+
.map(|requirement| {
309+
let expr: PhysicalSortExpr = requirement.to_owned().into();
310+
Ok(protobuf::PhysicalSortExprNode {
311+
expr: Some(Box::new(ctx.encode_expr(&expr.expr)?)),
312+
asc: !expr.options.descending,
313+
nulls_first: expr.options.nulls_first,
314+
})
315+
})
316+
.collect::<Result<Vec<_>>>()
317+
.map(|physical_sort_expr_nodes| {
318+
protobuf::PhysicalSortExprNodeCollection {
319+
physical_sort_expr_nodes,
320+
}
321+
})
322+
})
323+
.transpose()?;
324+
325+
self.sink()
326+
.try_to_proto(input, sort_order, self.schema().as_ref())
327+
}
271328
}
272329

273330
/// Create a output record batch with a count

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use std::vec;
2323
use arrow::array::RecordBatch;
2424
use arrow::csv::WriterBuilder;
2525
use arrow::datatypes::{Fields, TimeUnit};
26+
use async_trait::async_trait;
2627
use datafusion::arrow::array::ArrayRef;
2728
use datafusion::arrow::compute::kernels::sort::SortOptions;
2829
use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, SchemaRef};
@@ -39,7 +40,7 @@ use datafusion::datasource::physical_plan::{
3940
FileSinkConfig, ParquetSource, wrap_partition_type_in_dict,
4041
wrap_partition_value_in_dict,
4142
};
42-
use datafusion::datasource::sink::DataSinkExec;
43+
use datafusion::datasource::sink::{DataSink, DataSinkExec};
4344
use datafusion::datasource::source::DataSourceExec;
4445
use datafusion::execution::TaskContext;
4546
use datafusion::functions_aggregate::count::count_udaf;
@@ -2054,6 +2055,92 @@ fn roundtrip_explain() -> Result<()> {
20542055
Ok(())
20552056
}
20562057

2058+
#[derive(Debug)]
2059+
struct ProtoHookSink {
2060+
schema: SchemaRef,
2061+
}
2062+
2063+
impl DisplayAs for ProtoHookSink {
2064+
fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
2065+
write!(f, "ProtoHookSink")
2066+
}
2067+
}
2068+
2069+
#[async_trait]
2070+
impl DataSink for ProtoHookSink {
2071+
fn schema(&self) -> &SchemaRef {
2072+
&self.schema
2073+
}
2074+
2075+
async fn write_all(
2076+
&self,
2077+
_data: SendableRecordBatchStream,
2078+
_context: &Arc<TaskContext>,
2079+
) -> Result<u64> {
2080+
unreachable!("serialization test does not execute the sink")
2081+
}
2082+
2083+
fn try_to_proto(
2084+
&self,
2085+
input: PhysicalPlanNode,
2086+
sort_order: Option<protobuf::PhysicalSortExprNodeCollection>,
2087+
sink_schema: &Schema,
2088+
) -> Result<Option<PhysicalPlanNode>> {
2089+
assert!(matches!(
2090+
input.physical_plan_type,
2091+
Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))
2092+
));
2093+
assert_eq!(
2094+
sort_order
2095+
.as_ref()
2096+
.map(|ordering| ordering.physical_sort_expr_nodes.len()),
2097+
Some(1)
2098+
);
2099+
assert_eq!(sink_schema.fields().len(), 1);
2100+
2101+
Ok(Some(PhysicalPlanNode {
2102+
physical_plan_type: Some(
2103+
protobuf::physical_plan_node::PhysicalPlanType::Empty(
2104+
protobuf::EmptyExecNode {
2105+
schema: Some(sink_schema.try_into()?),
2106+
partitions: 1,
2107+
},
2108+
),
2109+
),
2110+
}))
2111+
}
2112+
}
2113+
2114+
#[test]
2115+
fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {
2116+
let input_schema = Arc::new(Schema::new(vec![Field::new(
2117+
"value",
2118+
DataType::Int64,
2119+
false,
2120+
)]));
2121+
let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&input_schema)));
2122+
let sink = Arc::new(ProtoHookSink {
2123+
schema: Arc::clone(&input_schema),
2124+
});
2125+
let sort_order = [PhysicalSortRequirement::new(
2126+
Arc::new(Column::new("value", 0)),
2127+
Some(SortOptions::default()),
2128+
)]
2129+
.into();
2130+
let plan = Arc::new(DataSinkExec::new(input, sink, Some(sort_order)));
2131+
2132+
let node = PhysicalPlanNode::try_from_physical_plan(
2133+
plan,
2134+
&DefaultPhysicalExtensionCodec {},
2135+
)?;
2136+
2137+
assert!(matches!(
2138+
node.physical_plan_type,
2139+
Some(protobuf::physical_plan_node::PhysicalPlanType::Empty(_))
2140+
));
2141+
Ok(())
2142+
}
2143+
20572144
#[tokio::test]
20582145
async fn roundtrip_json_source() -> Result<()> {
20592146
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)