Skip to content

Commit c74e81b

Browse files
committed
refactor(proto): move FileSinkConfig serde
FileSinkConfig is owned by datafusion-datasource, but its protobuf conversion lived in the central proto crate. Co-locating it lets sink implementations reuse the wire logic without depending on datafusion-proto. Keep existing TryFromProto implementations as compatibility delegates and verify both APIs produce the same protobuf data. CLOSES apache#23498
1 parent 7f41edb commit c74e81b

5 files changed

Lines changed: 204 additions & 91 deletions

File tree

‎datafusion/datasource/src/file_sink_config.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ use datafusion_expr::dml::InsertOp;
3232
use async_trait::async_trait;
3333
use object_store::ObjectStore;
3434

35+
#[cfg(feature = "proto")]
36+
mod proto;
37+
38+
#[cfg(feature = "proto")]
39+
pub use proto::parse_sink_sort_order;
40+
3541
/// Determines how `FileSink` output paths are interpreted.
3642
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
3743
pub enum FileOutputMode {
Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Protobuf conversion for the format-independent [`FileSinkConfig`].
19+
20+
use std::sync::Arc;
21+
22+
use arrow::compute::SortOptions;
23+
use arrow::datatypes::Schema;
24+
use datafusion_common::{Result, internal_datafusion_err};
25+
use datafusion_execution::object_store::ObjectStoreUrl;
26+
use datafusion_expr::dml::InsertOp;
27+
use datafusion_physical_expr::PhysicalSortExpr;
28+
use datafusion_physical_expr_common::sort_expr::LexRequirement;
29+
use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;
30+
use datafusion_proto_models::protobuf;
31+
32+
use crate::file_groups::FileGroup;
33+
use crate::file_sink_config::{FileOutputMode, FileSinkConfig};
34+
use crate::ListingTableUrl;
35+
36+
impl FileSinkConfig {
37+
/// Serialize this shared file-sink configuration without format-specific
38+
/// writer options.
39+
pub fn to_proto(&self) -> Result<protobuf::FileSinkConfig> {
40+
let file_groups = self
41+
.file_group
42+
.iter()
43+
.map(TryInto::try_into)
44+
.collect::<Result<Vec<_>>>()?;
45+
let table_paths = self
46+
.table_paths
47+
.iter()
48+
.map(ToString::to_string)
49+
.collect::<Vec<_>>();
50+
let table_partition_cols = self
51+
.table_partition_cols
52+
.iter()
53+
.map(|(name, data_type)| {
54+
Ok(protobuf::PartitionColumn {
55+
name: name.to_owned(),
56+
arrow_type: Some(data_type.try_into()?),
57+
})
58+
})
59+
.collect::<Result<Vec<_>>>()?;
60+
let file_output_mode = match self.file_output_mode {
61+
FileOutputMode::Automatic => protobuf::FileOutputMode::Automatic,
62+
FileOutputMode::SingleFile => protobuf::FileOutputMode::SingleFile,
63+
FileOutputMode::Directory => protobuf::FileOutputMode::Directory,
64+
};
65+
66+
Ok(protobuf::FileSinkConfig {
67+
object_store_url: self.object_store_url.to_string(),
68+
file_groups,
69+
table_paths,
70+
output_schema: Some(self.output_schema.as_ref().try_into()?),
71+
table_partition_cols,
72+
keep_partition_by_columns: self.keep_partition_by_columns,
73+
insert_op: self.insert_op as i32,
74+
file_extension: self.file_extension.clone(),
75+
file_output_mode: file_output_mode.into(),
76+
})
77+
}
78+
79+
/// Reconstruct a shared file-sink configuration from protobuf.
80+
pub fn from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self> {
81+
let file_group = FileGroup::new(
82+
conf.file_groups
83+
.iter()
84+
.map(TryInto::try_into)
85+
.collect::<Result<Vec<_>>>()?,
86+
);
87+
let table_paths = conf
88+
.table_paths
89+
.iter()
90+
.map(ListingTableUrl::parse)
91+
.collect::<Result<Vec<_>>>()?;
92+
let table_partition_cols = conf
93+
.table_partition_cols
94+
.iter()
95+
.map(|protobuf::PartitionColumn { name, arrow_type }| {
96+
let data_type = arrow_type
97+
.as_ref()
98+
.ok_or_else(|| {
99+
internal_datafusion_err!(
100+
"PartitionColumn is missing required field 'arrow_type'"
101+
)
102+
})?
103+
.try_into()?;
104+
Ok((name.clone(), data_type))
105+
})
106+
.collect::<Result<Vec<_>>>()?;
107+
let insert_op = match conf.insert_op() {
108+
protobuf::InsertOp::Append => InsertOp::Append,
109+
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
110+
protobuf::InsertOp::Replace => InsertOp::Replace,
111+
};
112+
let file_output_mode = match conf.file_output_mode() {
113+
protobuf::FileOutputMode::Automatic => FileOutputMode::Automatic,
114+
protobuf::FileOutputMode::SingleFile => FileOutputMode::SingleFile,
115+
protobuf::FileOutputMode::Directory => FileOutputMode::Directory,
116+
};
117+
let output_schema = conf.output_schema.as_ref().ok_or_else(|| {
118+
internal_datafusion_err!(
119+
"FileSinkConfig is missing required field 'output_schema'"
120+
)
121+
})?;
122+
123+
Ok(Self {
124+
original_url: String::default(),
125+
object_store_url: ObjectStoreUrl::parse(&conf.object_store_url)?,
126+
file_group,
127+
table_paths,
128+
output_schema: Arc::new(output_schema.try_into()?),
129+
table_partition_cols,
130+
insert_op,
131+
keep_partition_by_columns: conf.keep_partition_by_columns,
132+
file_extension: conf.file_extension.clone(),
133+
file_output_mode,
134+
})
135+
}
136+
}
137+
138+
/// Decode a sink's optional required output ordering against its input schema.
139+
pub fn parse_sink_sort_order(
140+
collection: Option<&protobuf::PhysicalSortExprNodeCollection>,
141+
ctx: &ExecutionPlanDecodeCtx<'_>,
142+
schema: &Schema,
143+
) -> Result<Option<LexRequirement>> {
144+
let Some(collection) = collection else {
145+
return Ok(None);
146+
};
147+
let sort_exprs = collection
148+
.physical_sort_expr_nodes
149+
.iter()
150+
.map(|node| {
151+
let expr = node.expr.as_ref().ok_or_else(|| {
152+
internal_datafusion_err!("Unexpected empty physical expression")
153+
})?;
154+
Ok(PhysicalSortExpr {
155+
expr: ctx.decode_expr(expr, schema)?,
156+
options: SortOptions {
157+
descending: !node.asc,
158+
nulls_first: node.nulls_first,
159+
},
160+
})
161+
})
162+
.collect::<Result<Vec<_>>>()?;
163+
Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
164+
}

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,14 @@ use datafusion_datasource::file::FileSource;
3030
use datafusion_datasource::file_groups::FileGroup;
3131
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
3232
use datafusion_datasource::file_sink_config::FileSinkConfig;
33-
use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile, TableSchema};
33+
use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};
3434
use datafusion_datasource_csv::file_format::CsvSink;
3535
use datafusion_datasource_json::file_format::JsonSink;
3636
#[cfg(feature = "parquet")]
3737
use datafusion_datasource_parquet::file_format::ParquetSink;
3838
use datafusion_execution::object_store::ObjectStoreUrl;
3939
use datafusion_execution::{FunctionRegistry, TaskContext};
4040
use datafusion_expr::WindowFunctionDefinition;
41-
use datafusion_expr::dml::InsertOp;
4241
use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};
4342
use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
4443
use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
@@ -694,53 +693,7 @@ impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
694693
type Error = DataFusionError;
695694

696695
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
697-
let file_group = FileGroup::new(
698-
conf.file_groups
699-
.iter()
700-
.map(TryInto::try_into)
701-
.collect::<Result<Vec<_>>>()?,
702-
);
703-
let table_paths = conf
704-
.table_paths
705-
.iter()
706-
.map(ListingTableUrl::parse)
707-
.collect::<Result<Vec<_>>>()?;
708-
let table_partition_cols = conf
709-
.table_partition_cols
710-
.iter()
711-
.map(|protobuf::PartitionColumn { name, arrow_type }| {
712-
let data_type = convert_required!(arrow_type)?;
713-
Ok((name.clone(), data_type))
714-
})
715-
.collect::<Result<Vec<_>>>()?;
716-
let insert_op = match conf.insert_op() {
717-
protobuf::InsertOp::Append => InsertOp::Append,
718-
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
719-
protobuf::InsertOp::Replace => InsertOp::Replace,
720-
};
721-
let file_output_mode = match conf.file_output_mode() {
722-
protobuf::FileOutputMode::Automatic => {
723-
datafusion_datasource::file_sink_config::FileOutputMode::Automatic
724-
}
725-
protobuf::FileOutputMode::SingleFile => {
726-
datafusion_datasource::file_sink_config::FileOutputMode::SingleFile
727-
}
728-
protobuf::FileOutputMode::Directory => {
729-
datafusion_datasource::file_sink_config::FileOutputMode::Directory
730-
}
731-
};
732-
Ok(Self {
733-
original_url: String::default(),
734-
object_store_url: ObjectStoreUrl::parse(&conf.object_store_url)?,
735-
file_group,
736-
table_paths,
737-
output_schema: Arc::new(convert_required!(conf.output_schema)?),
738-
table_partition_cols,
739-
insert_op,
740-
keep_partition_by_columns: conf.keep_partition_by_columns,
741-
file_extension: conf.file_extension.clone(),
742-
file_output_mode,
743-
})
696+
FileSinkConfig::from_proto(conf)
744697
}
745698
}
746699

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 1 addition & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -611,47 +611,6 @@ impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {
611611
type Error = DataFusionError;
612612

613613
fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {
614-
let file_groups = conf
615-
.file_group
616-
.iter()
617-
.map(protobuf::PartitionedFile::try_from_proto)
618-
.collect::<Result<Vec<_>>>()?;
619-
let table_paths = conf
620-
.table_paths
621-
.iter()
622-
.map(ToString::to_string)
623-
.collect::<Vec<_>>();
624-
let table_partition_cols = conf
625-
.table_partition_cols
626-
.iter()
627-
.map(|(name, data_type)| {
628-
Ok(protobuf::PartitionColumn {
629-
name: name.to_owned(),
630-
arrow_type: Some(data_type.try_into()?),
631-
})
632-
})
633-
.collect::<Result<Vec<_>>>()?;
634-
let file_output_mode = match conf.file_output_mode {
635-
datafusion_datasource::file_sink_config::FileOutputMode::Automatic => {
636-
protobuf::FileOutputMode::Automatic
637-
}
638-
datafusion_datasource::file_sink_config::FileOutputMode::SingleFile => {
639-
protobuf::FileOutputMode::SingleFile
640-
}
641-
datafusion_datasource::file_sink_config::FileOutputMode::Directory => {
642-
protobuf::FileOutputMode::Directory
643-
}
644-
};
645-
Ok(Self {
646-
object_store_url: conf.object_store_url.to_string(),
647-
file_groups,
648-
table_paths,
649-
output_schema: Some(conf.output_schema.as_ref().try_into()?),
650-
table_partition_cols,
651-
keep_partition_by_columns: conf.keep_partition_by_columns,
652-
insert_op: conf.insert_op as i32,
653-
file_extension: conf.file_extension.to_string(),
654-
file_output_mode: file_output_mode.into(),
655-
})
614+
conf.to_proto()
656615
}
657616
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ use datafusion_proto::bytes::{
134134
physical_plan_from_bytes_with_proto_converter,
135135
physical_plan_to_bytes_with_proto_converter,
136136
};
137+
use datafusion_proto::convert::TryFromProto;
137138
use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter;
138139
use datafusion_proto::physical_plan::{
139140
AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec,
@@ -2141,6 +2142,36 @@ fn data_sink_exec_delegates_to_sink_proto_hook() -> Result<()> {
21412142
Ok(())
21422143
}
21432144

2145+
#[test]
2146+
fn file_sink_config_conversion_preserves_compatibility_api() -> Result<()> {
2147+
let schema = Arc::new(Schema::new(vec![Field::new(
2148+
"partition",
2149+
DataType::Utf8,
2150+
false,
2151+
)]));
2152+
let config = FileSinkConfig {
2153+
original_url: "file:///tmp/output".to_string(),
2154+
object_store_url: ObjectStoreUrl::local_filesystem(),
2155+
file_group: FileGroup::new(vec![PartitionedFile::new("/tmp/output", 1)]),
2156+
table_paths: vec![ListingTableUrl::parse("file:///tmp/output")?],
2157+
output_schema: schema,
2158+
table_partition_cols: vec![("partition".to_string(), DataType::Utf8)],
2159+
insert_op: InsertOp::Overwrite,
2160+
keep_partition_by_columns: true,
2161+
file_extension: "parquet".to_string(),
2162+
file_output_mode: FileOutputMode::Directory,
2163+
};
2164+
2165+
let direct = config.to_proto()?;
2166+
let compatibility = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167+
assert_eq!(direct, compatibility);
2168+
2169+
let direct = FileSinkConfig::from_proto(&direct)?;
2170+
let compatibility = FileSinkConfig::try_from_proto(&compatibility)?;
2171+
assert_eq!(direct.to_proto()?, compatibility.to_proto()?);
2172+
Ok(())
2173+
}
2174+
21442175
#[tokio::test]
21452176
async fn roundtrip_json_source() -> Result<()> {
21462177
let ctx = SessionContext::new();

0 commit comments

Comments
 (0)