Skip to content

Commit 92fe98b

Browse files
committed
refactor(proto): refine data sink serde hooks
Defer encoding until a sink opts into serialization so the legacy fallback does not encode child plans and sort expressions twice. Restore FileSinkConfig's standard TryFrom API and map enum values by name to preserve wire compatibility independently of discriminants. Refs apache#23498
1 parent a9e19cf commit 92fe98b

6 files changed

Lines changed: 110 additions & 106 deletions

File tree

‎datafusion/datasource/src/file_sink_config.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,6 @@ use object_store::ObjectStore;
3535
#[cfg(feature = "proto")]
3636
mod proto;
3737

38-
#[cfg(feature = "proto")]
39-
pub use proto::parse_sink_sort_order;
40-
4138
/// Determines how `FileSink` output paths are interpreted.
4239
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4340
pub enum FileOutputMode {

‎datafusion/datasource/src/file_sink_config/proto.rs‎

Lines changed: 26 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,35 +19,32 @@
1919
2020
use std::sync::Arc;
2121

22-
use arrow::compute::SortOptions;
23-
use arrow::datatypes::Schema;
24-
use datafusion_common::{Result, internal_datafusion_err};
22+
use datafusion_common::{DataFusionError, Result, internal_datafusion_err};
2523
use datafusion_execution::object_store::ObjectStoreUrl;
2624
use datafusion_expr::dml::InsertOp;
27-
use datafusion_physical_expr::PhysicalSortExpr;
28-
use datafusion_physical_expr_common::sort_expr::LexRequirement;
29-
use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;
3025
use datafusion_proto_models::protobuf;
3126

3227
use crate::ListingTableUrl;
3328
use crate::file_groups::FileGroup;
3429
use crate::file_sink_config::{FileOutputMode, FileSinkConfig};
3530

36-
impl FileSinkConfig {
31+
impl TryFrom<&FileSinkConfig> for protobuf::FileSinkConfig {
32+
type Error = DataFusionError;
33+
3734
/// Serialize this shared file-sink configuration without format-specific
3835
/// writer options.
39-
pub fn to_proto(&self) -> Result<protobuf::FileSinkConfig> {
40-
let file_groups = self
36+
fn try_from(config: &FileSinkConfig) -> Result<Self> {
37+
let file_groups = config
4138
.file_group
4239
.iter()
4340
.map(TryInto::try_into)
4441
.collect::<Result<Vec<_>>>()?;
45-
let table_paths = self
42+
let table_paths = config
4643
.table_paths
4744
.iter()
4845
.map(ToString::to_string)
4946
.collect::<Vec<_>>();
50-
let table_partition_cols = self
47+
let table_partition_cols = config
5148
.table_partition_cols
5249
.iter()
5350
.map(|(name, data_type)| {
@@ -57,27 +54,36 @@ impl FileSinkConfig {
5754
})
5855
})
5956
.collect::<Result<Vec<_>>>()?;
60-
let file_output_mode = match self.file_output_mode {
57+
let insert_op = match config.insert_op {
58+
InsertOp::Append => protobuf::InsertOp::Append,
59+
InsertOp::Overwrite => protobuf::InsertOp::Overwrite,
60+
InsertOp::Replace => protobuf::InsertOp::Replace,
61+
};
62+
let file_output_mode = match config.file_output_mode {
6163
FileOutputMode::Automatic => protobuf::FileOutputMode::Automatic,
6264
FileOutputMode::SingleFile => protobuf::FileOutputMode::SingleFile,
6365
FileOutputMode::Directory => protobuf::FileOutputMode::Directory,
6466
};
6567

6668
Ok(protobuf::FileSinkConfig {
67-
object_store_url: self.object_store_url.to_string(),
69+
object_store_url: config.object_store_url.to_string(),
6870
file_groups,
6971
table_paths,
70-
output_schema: Some(self.output_schema.as_ref().try_into()?),
72+
output_schema: Some(config.output_schema.as_ref().try_into()?),
7173
table_partition_cols,
72-
keep_partition_by_columns: self.keep_partition_by_columns,
73-
insert_op: self.insert_op as i32,
74-
file_extension: self.file_extension.clone(),
74+
keep_partition_by_columns: config.keep_partition_by_columns,
75+
insert_op: insert_op.into(),
76+
file_extension: config.file_extension.clone(),
7577
file_output_mode: file_output_mode.into(),
7678
})
7779
}
80+
}
81+
82+
impl TryFrom<&protobuf::FileSinkConfig> for FileSinkConfig {
83+
type Error = DataFusionError;
7884

7985
/// Reconstruct a shared file-sink configuration from protobuf.
80-
pub fn from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self> {
86+
fn try_from(conf: &protobuf::FileSinkConfig) -> Result<Self> {
8187
let file_group = FileGroup::new(
8288
conf.file_groups
8389
.iter()
@@ -148,36 +154,9 @@ impl FileSinkConfig {
148154
}
149155
}
150156

151-
/// Decode a sink's optional required output ordering against its input schema.
152-
pub fn parse_sink_sort_order(
153-
collection: Option<&protobuf::PhysicalSortExprNodeCollection>,
154-
ctx: &ExecutionPlanDecodeCtx<'_>,
155-
schema: &Schema,
156-
) -> Result<Option<LexRequirement>> {
157-
let Some(collection) = collection else {
158-
return Ok(None);
159-
};
160-
let sort_exprs = collection
161-
.physical_sort_expr_nodes
162-
.iter()
163-
.map(|node| {
164-
let expr = node.expr.as_ref().ok_or_else(|| {
165-
internal_datafusion_err!("Unexpected empty physical expression")
166-
})?;
167-
Ok(PhysicalSortExpr {
168-
expr: ctx.decode_expr(expr, schema)?,
169-
options: SortOptions {
170-
descending: !node.asc,
171-
nulls_first: node.nulls_first,
172-
},
173-
})
174-
})
175-
.collect::<Result<Vec<_>>>()?;
176-
Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
177-
}
178157
#[cfg(test)]
179158
mod tests {
180-
use datafusion_common::DataFusionError;
159+
use arrow::datatypes::Schema;
181160

182161
use super::*;
183162

@@ -203,7 +182,7 @@ mod tests {
203182
mutate(&mut conf);
204183

205184
let error =
206-
FileSinkConfig::from_proto(&conf).expect_err("invalid config should fail");
185+
FileSinkConfig::try_from(&conf).expect_err("invalid config should fail");
207186
match error {
208187
DataFusionError::Internal(message) => {
209188
let message = message

‎datafusion/datasource/src/sink.rs‎

Lines changed: 41 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -74,19 +74,16 @@ pub trait DataSink: Any + DisplayAs + Debug + Send + Sync {
7474

7575
/// Serialize this sink into a full protobuf plan node, if it knows how.
7676
///
77-
/// [`DataSinkExec::try_to_proto`] encodes the shared child plan and required
78-
/// output ordering before delegating to this hook. Implementations only need
79-
/// to add their sink-specific fields.
77+
/// Implementations can use `ctx` to encode the input plan, sink-specific
78+
/// expressions, and [`DataSinkExec::encode_sort_order`].
8079
///
81-
/// Returning `Ok(None)` preserves the legacy central serialization fallback.
80+
/// Returning `Ok(None)` preserves the legacy central serialization fallback
81+
/// without eagerly encoding any child plans or expressions.
8282
#[cfg(feature = "proto")]
8383
fn try_to_proto(
8484
&self,
85-
_input: datafusion_proto_models::protobuf::PhysicalPlanNode,
86-
_sort_order: Option<
87-
datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection,
88-
>,
89-
_sink_schema: &Schema,
85+
_exec: &DataSinkExec,
86+
_ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
9087
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
9188
Ok(None)
9289
}
@@ -164,6 +161,39 @@ impl DataSinkExec {
164161
&self.sort_order
165162
}
166163

164+
/// Encode the optional sink ordering for a protobuf plan node.
165+
#[cfg(feature = "proto")]
166+
pub fn encode_sort_order(
167+
&self,
168+
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
169+
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalSortExprNodeCollection>>
170+
{
171+
use datafusion_physical_expr::PhysicalSortExpr;
172+
use datafusion_proto_models::protobuf;
173+
174+
self.sort_order
175+
.as_ref()
176+
.map(|requirements| {
177+
requirements
178+
.iter()
179+
.map(|requirement| {
180+
let expr: PhysicalSortExpr = requirement.to_owned().into();
181+
Ok(protobuf::PhysicalSortExprNode {
182+
expr: Some(Box::new(ctx.encode_expr(&expr.expr)?)),
183+
asc: !expr.options.descending,
184+
nulls_first: expr.options.nulls_first,
185+
})
186+
})
187+
.collect::<Result<Vec<_>>>()
188+
.map(|physical_sort_expr_nodes| {
189+
protobuf::PhysicalSortExprNodeCollection {
190+
physical_sort_expr_nodes,
191+
}
192+
})
193+
})
194+
.transpose()
195+
}
196+
167197
fn create_schema(
168198
input: &Arc<dyn ExecutionPlan>,
169199
schema: SchemaRef,
@@ -288,42 +318,13 @@ impl ExecutionPlan for DataSinkExec {
288318
self.sink.metrics()
289319
}
290320

291-
/// Encodes the shared sink plan fields before delegating the sink-specific
292-
/// protobuf representation to [`DataSink::try_to_proto`].
321+
/// Delegates protobuf serialization to the underlying sink.
293322
#[cfg(feature = "proto")]
294323
fn try_to_proto(
295324
&self,
296325
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
297326
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
298-
use datafusion_physical_expr::PhysicalSortExpr;
299-
use datafusion_proto_models::protobuf;
300-
301-
let input = ctx.encode_child(self.input())?;
302-
let sort_order = self
303-
.sort_order()
304-
.as_ref()
305-
.map(|requirements| {
306-
requirements
307-
.iter()
308-
.map(|requirement| {
309-
let expr: PhysicalSortExpr = requirement.to_owned().into();
310-
Ok(protobuf::PhysicalSortExprNode {
311-
expr: Some(Box::new(ctx.encode_expr(&expr.expr)?)),
312-
asc: !expr.options.descending,
313-
nulls_first: expr.options.nulls_first,
314-
})
315-
})
316-
.collect::<Result<Vec<_>>>()
317-
.map(|physical_sort_expr_nodes| {
318-
protobuf::PhysicalSortExprNodeCollection {
319-
physical_sort_expr_nodes,
320-
}
321-
})
322-
})
323-
.transpose()?;
324-
325-
self.sink()
326-
.try_to_proto(input, sort_order, self.schema().as_ref())
327+
self.sink().try_to_proto(self, ctx)
327328
}
328329
}
329330

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,7 @@ impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
620620
type Error = DataFusionError;
621621

622622
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
623-
FileSinkConfig::from_proto(conf)
623+
conf.try_into()
624624
}
625625
}
626626

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,6 +552,6 @@ impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {
552552
type Error = DataFusionError;
553553

554554
fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {
555-
conf.to_proto()
555+
conf.try_into()
556556
}
557557
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};
8282
use datafusion::physical_plan::metrics::MetricCategory;
8383
use datafusion::physical_plan::placeholder_row::PlaceholderRowExec;
8484
use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};
85+
use datafusion::physical_plan::proto::ExecutionPlanEncodeCtx;
8586
use datafusion::physical_plan::repartition::RepartitionExec;
8687
use datafusion::physical_plan::scalar_subquery::{
8788
ScalarSubqueryExec, ScalarSubqueryLink,
@@ -134,7 +135,6 @@ use datafusion_proto::bytes::{
134135
physical_plan_from_bytes_with_proto_converter,
135136
physical_plan_to_bytes_with_proto_converter,
136137
};
137-
use datafusion_proto::convert::TryFromProto;
138138
use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
139139
use datafusion_proto::physical_plan::{
140140
AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,
@@ -2083,10 +2083,11 @@ impl DataSink for ProtoHookSink {
20832083

20842084
fn try_to_proto(
20852085
&self,
2086-
input: PhysicalPlanNode,
2087-
sort_order: Option<protobuf::PhysicalSortExprNodeCollection>,
2088-
sink_schema: &Schema,
2086+
exec: &DataSinkExec,
2087+
ctx: &ExecutionPlanEncodeCtx<'_>,
20892088
) -> Result<Option<PhysicalPlanNode>> {
2089+
let input = ctx.encode_child(exec.input())?;
2090+
let sort_order = exec.encode_sort_order(ctx)?;
20902091
assert!(matches!(
20912092
input.physical_plan_type,
20922093
Some(protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow(_))
@@ -2097,13 +2098,13 @@ impl DataSink for ProtoHookSink {
20972098
.map(|ordering| ordering.physical_sort_expr_nodes.len()),
20982099
Some(1)
20992100
);
2100-
assert_eq!(sink_schema.fields().len(), 1);
2101+
assert_eq!(exec.schema().fields().len(), 1);
21012102

21022103
Ok(Some(PhysicalPlanNode {
21032104
physical_plan_type: Some(
21042105
protobuf::physical_plan_node::PhysicalPlanType::Empty(
21052106
protobuf::EmptyExecNode {
2106-
schema: Some(sink_schema.try_into()?),
2107+
schema: Some(exec.schema().as_ref().try_into()?),
21072108
partitions: 1,
21082109
},
21092110
),
@@ -2143,7 +2144,7 @@ fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {
21432144
}
21442145

21452146
#[test]
2146-
fn file_sink_config_conversion_preserves_compatibility_api() -> Result<()> {
2147+
fn file_sink_config_roundtrip_preserves_fields() -> Result<()> {
21472148
let schema = Arc::new(Schema::new(vec![Field::new(
21482149
"partition",
21492150
DataType::Utf8,
@@ -2162,14 +2163,40 @@ fn file_sink_config_conversion_preserves_compatibility_api() -> Result<()> {
21622163
file_output_mode: FileOutputMode::Directory,
21632164
};
21642165

2165-
let encoded = config.to_proto()?;
2166-
let compatibility_encoded = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167-
assert_eq!(encoded, compatibility_encoded);
2166+
let encoded = protobuf::FileSinkConfig::try_from(&config)?;
2167+
assert_eq!(encoded.insert_op(), protobuf::InsertOp::Overwrite);
2168+
assert_eq!(
2169+
encoded.file_output_mode(),
2170+
protobuf::FileOutputMode::Directory
2171+
);
21682172

2169-
let decoded = FileSinkConfig::from_proto(&encoded)?;
2170-
let compatibility_decoded = FileSinkConfig::try_from_proto(&encoded)?;
2171-
assert_eq!(decoded.to_proto()?, encoded);
2172-
assert_eq!(compatibility_decoded.to_proto()?, encoded);
2173+
let decoded = FileSinkConfig::try_from(&encoded)?;
2174+
assert_eq!(decoded.object_store_url, config.object_store_url);
2175+
assert_eq!(decoded.table_paths, config.table_paths);
2176+
assert_eq!(
2177+
decoded.output_schema.as_ref(),
2178+
config.output_schema.as_ref()
2179+
);
2180+
assert_eq!(decoded.table_partition_cols, config.table_partition_cols);
2181+
assert_eq!(decoded.insert_op, config.insert_op);
2182+
assert_eq!(
2183+
decoded.keep_partition_by_columns,
2184+
config.keep_partition_by_columns
2185+
);
2186+
assert_eq!(decoded.file_extension, config.file_extension);
2187+
assert_eq!(decoded.file_output_mode, config.file_output_mode);
2188+
2189+
let [decoded_file] = decoded.file_group.files() else {
2190+
panic!("expected one decoded output file");
2191+
};
2192+
let [config_file] = config.file_group.files() else {
2193+
panic!("expected one configured output file");
2194+
};
2195+
assert_eq!(
2196+
decoded_file.object_meta.location,
2197+
config_file.object_meta.location
2198+
);
2199+
assert_eq!(decoded_file.object_meta.size, config_file.object_meta.size);
21732200
Ok(())
21742201
}
21752202

0 commit comments

Comments
 (0)