Skip to content

Commit 457c647

Browse files
committed
refactor(proto): move FileSinkConfig serde
FileSinkConfig is owned by datafusion-datasource, but its protobuf conversion lived in the central proto crate. Co-locating it lets sink implementations reuse the wire logic without depending on datafusion-proto. Keep existing TryFromProto implementations as compatibility delegates and verify both APIs produce the same protobuf data. CLOSES apache#23498
1 parent 0c4f9f6 commit 457c647

5 files changed

Lines changed: 272 additions & 91 deletions

File tree

‎datafusion/datasource/src/file_sink_config.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ use datafusion_expr::dml::InsertOp;
3232
use async_trait::async_trait;
3333
use object_store::ObjectStore;
3434

35+
#[cfg(feature = "proto")]
36+
mod proto;
37+
38+
#[cfg(feature = "proto")]
39+
pub use proto::parse_sink_sort_order;
40+
3541
/// Determines how `FileSink` output paths are interpreted.
3642
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
3743
pub enum FileOutputMode {
Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Protobuf conversion for the format-independent [`FileSinkConfig`].
19+
20+
use std::sync::Arc;
21+
22+
use arrow::compute::SortOptions;
23+
use arrow::datatypes::Schema;
24+
use chrono::{TimeZone, Utc};
25+
use datafusion_common::{DataFusionError, Result, internal_datafusion_err};
26+
use datafusion_execution::object_store::ObjectStoreUrl;
27+
use datafusion_expr::dml::InsertOp;
28+
use datafusion_physical_expr::PhysicalSortExpr;
29+
use datafusion_physical_expr_common::sort_expr::LexRequirement;
30+
use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;
31+
use datafusion_proto_models::protobuf;
32+
use object_store::ObjectMeta;
33+
use object_store::path::Path;
34+
35+
use crate::file_groups::FileGroup;
36+
use crate::file_sink_config::{FileOutputMode, FileSinkConfig};
37+
use crate::{ListingTableUrl, PartitionedFile};
38+
39+
impl FileSinkConfig {
40+
/// Serialize this shared file-sink configuration without format-specific
41+
/// writer options.
42+
pub fn to_proto(&self) -> Result<protobuf::FileSinkConfig> {
43+
let file_groups = self
44+
.file_group
45+
.iter()
46+
.map(partitioned_file_to_proto)
47+
.collect::<Result<Vec<_>>>()?;
48+
let table_paths = self
49+
.table_paths
50+
.iter()
51+
.map(ToString::to_string)
52+
.collect::<Vec<_>>();
53+
let table_partition_cols = self
54+
.table_partition_cols
55+
.iter()
56+
.map(|(name, data_type)| {
57+
Ok(protobuf::PartitionColumn {
58+
name: name.to_owned(),
59+
arrow_type: Some(data_type.try_into()?),
60+
})
61+
})
62+
.collect::<Result<Vec<_>>>()?;
63+
let file_output_mode = match self.file_output_mode {
64+
FileOutputMode::Automatic => protobuf::FileOutputMode::Automatic,
65+
FileOutputMode::SingleFile => protobuf::FileOutputMode::SingleFile,
66+
FileOutputMode::Directory => protobuf::FileOutputMode::Directory,
67+
};
68+
69+
Ok(protobuf::FileSinkConfig {
70+
object_store_url: self.object_store_url.to_string(),
71+
file_groups,
72+
table_paths,
73+
output_schema: Some(self.output_schema.as_ref().try_into()?),
74+
table_partition_cols,
75+
keep_partition_by_columns: self.keep_partition_by_columns,
76+
insert_op: self.insert_op as i32,
77+
file_extension: self.file_extension.clone(),
78+
file_output_mode: file_output_mode.into(),
79+
})
80+
}
81+
82+
/// Reconstruct a shared file-sink configuration from protobuf.
83+
pub fn from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self> {
84+
let file_group = FileGroup::new(
85+
conf.file_groups
86+
.iter()
87+
.map(partitioned_file_from_proto)
88+
.collect::<Result<Vec<_>>>()?,
89+
);
90+
let table_paths = conf
91+
.table_paths
92+
.iter()
93+
.map(ListingTableUrl::parse)
94+
.collect::<Result<Vec<_>>>()?;
95+
let table_partition_cols = conf
96+
.table_partition_cols
97+
.iter()
98+
.map(|protobuf::PartitionColumn { name, arrow_type }| {
99+
let data_type = arrow_type
100+
.as_ref()
101+
.ok_or_else(|| {
102+
internal_datafusion_err!(
103+
"PartitionColumn is missing required field 'arrow_type'"
104+
)
105+
})?
106+
.try_into()?;
107+
Ok((name.clone(), data_type))
108+
})
109+
.collect::<Result<Vec<_>>>()?;
110+
let insert_op = match conf.insert_op() {
111+
protobuf::InsertOp::Append => InsertOp::Append,
112+
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
113+
protobuf::InsertOp::Replace => InsertOp::Replace,
114+
};
115+
let file_output_mode = match conf.file_output_mode() {
116+
protobuf::FileOutputMode::Automatic => FileOutputMode::Automatic,
117+
protobuf::FileOutputMode::SingleFile => FileOutputMode::SingleFile,
118+
protobuf::FileOutputMode::Directory => FileOutputMode::Directory,
119+
};
120+
let output_schema = conf.output_schema.as_ref().ok_or_else(|| {
121+
internal_datafusion_err!(
122+
"FileSinkConfig is missing required field 'output_schema'"
123+
)
124+
})?;
125+
126+
Ok(Self {
127+
original_url: String::default(),
128+
object_store_url: ObjectStoreUrl::parse(&conf.object_store_url)?,
129+
file_group,
130+
table_paths,
131+
output_schema: Arc::new(output_schema.try_into()?),
132+
table_partition_cols,
133+
insert_op,
134+
keep_partition_by_columns: conf.keep_partition_by_columns,
135+
file_extension: conf.file_extension.clone(),
136+
file_output_mode,
137+
})
138+
}
139+
}
140+
141+
/// Decode a sink's optional required output ordering against its input schema.
142+
pub fn parse_sink_sort_order(
143+
collection: Option<&protobuf::PhysicalSortExprNodeCollection>,
144+
ctx: &ExecutionPlanDecodeCtx<'_>,
145+
schema: &Schema,
146+
) -> Result<Option<LexRequirement>> {
147+
let Some(collection) = collection else {
148+
return Ok(None);
149+
};
150+
let sort_exprs = collection
151+
.physical_sort_expr_nodes
152+
.iter()
153+
.map(|node| {
154+
let expr = node.expr.as_ref().ok_or_else(|| {
155+
internal_datafusion_err!("Unexpected empty physical expression")
156+
})?;
157+
Ok(PhysicalSortExpr {
158+
expr: ctx.decode_expr(expr, schema)?,
159+
options: SortOptions {
160+
descending: !node.asc,
161+
nulls_first: node.nulls_first,
162+
},
163+
})
164+
})
165+
.collect::<Result<Vec<_>>>()?;
166+
Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
167+
}
168+
169+
fn partitioned_file_to_proto(
170+
file: &PartitionedFile,
171+
) -> Result<protobuf::PartitionedFile> {
172+
let last_modified = file.object_meta.last_modified;
173+
let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| {
174+
DataFusionError::Plan(format!(
175+
"Invalid timestamp on PartitionedFile::ObjectMeta: {last_modified}"
176+
))
177+
})? as u64;
178+
179+
Ok(protobuf::PartitionedFile {
180+
arrow_schema: file
181+
.arrow_schema
182+
.as_ref()
183+
.map(|schema| schema.as_ref().try_into())
184+
.transpose()?,
185+
path: file.object_meta.location.as_ref().to_owned(),
186+
size: file.object_meta.size,
187+
last_modified_ns,
188+
partition_values: file
189+
.partition_values
190+
.iter()
191+
.map(TryInto::try_into)
192+
.collect::<Result<Vec<_>, _>>()?,
193+
range: file.range.as_ref().map(|range| protobuf::FileRange {
194+
start: range.start,
195+
end: range.end,
196+
}),
197+
statistics: file.statistics.as_ref().map(|stats| stats.as_ref().into()),
198+
})
199+
}
200+
201+
fn partitioned_file_from_proto(
202+
file: &protobuf::PartitionedFile,
203+
) -> Result<PartitionedFile> {
204+
let mut partitioned_file = PartitionedFile::new_from_meta(ObjectMeta {
205+
location: Path::parse(file.path.as_str()).map_err(|error| {
206+
internal_datafusion_err!("Invalid object_store path: {error}")
207+
})?,
208+
last_modified: Utc.timestamp_nanos(file.last_modified_ns as i64),
209+
size: file.size,
210+
e_tag: None,
211+
version: None,
212+
})
213+
.with_partition_values(
214+
file.partition_values
215+
.iter()
216+
.map(TryInto::try_into)
217+
.collect::<Result<Vec<_>, _>>()?,
218+
);
219+
if let Some(schema) = file.arrow_schema.as_ref() {
220+
partitioned_file = partitioned_file.with_arrow_schema(Arc::new(
221+
schema.try_into().map_err(DataFusionError::from)?,
222+
));
223+
}
224+
if let Some(range) = file.range.as_ref() {
225+
partitioned_file = partitioned_file.with_range(range.start, range.end);
226+
}
227+
if let Some(statistics) = file.statistics.as_ref() {
228+
partitioned_file =
229+
partitioned_file.with_statistics(Arc::new(statistics.try_into()?));
230+
}
231+
Ok(partitioned_file)
232+
}

‎datafusion/proto/src/physical_plan/from_proto.rs‎

Lines changed: 2 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,14 @@ use datafusion_datasource::file::FileSource;
3131
use datafusion_datasource::file_groups::FileGroup;
3232
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
3333
use datafusion_datasource::file_sink_config::FileSinkConfig;
34-
use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile, TableSchema};
34+
use datafusion_datasource::{FileRange, PartitionedFile, TableSchema};
3535
use datafusion_datasource_csv::file_format::CsvSink;
3636
use datafusion_datasource_json::file_format::JsonSink;
3737
#[cfg(feature = "parquet")]
3838
use datafusion_datasource_parquet::file_format::ParquetSink;
3939
use datafusion_execution::object_store::ObjectStoreUrl;
4040
use datafusion_execution::{FunctionRegistry, TaskContext};
4141
use datafusion_expr::WindowFunctionDefinition;
42-
use datafusion_expr::dml::InsertOp;
4342
use datafusion_physical_expr::expressions::{LambdaExpr, LambdaVariable};
4443
use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
4544
use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
@@ -728,53 +727,7 @@ impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig {
728727
type Error = DataFusionError;
729728

730729
fn try_from_proto(conf: &protobuf::FileSinkConfig) -> Result<Self, Self::Error> {
731-
let file_group = FileGroup::new(
732-
conf.file_groups
733-
.iter()
734-
.map(PartitionedFile::try_from_proto)
735-
.collect::<Result<Vec<_>>>()?,
736-
);
737-
let table_paths = conf
738-
.table_paths
739-
.iter()
740-
.map(ListingTableUrl::parse)
741-
.collect::<Result<Vec<_>>>()?;
742-
let table_partition_cols = conf
743-
.table_partition_cols
744-
.iter()
745-
.map(|protobuf::PartitionColumn { name, arrow_type }| {
746-
let data_type = convert_required!(arrow_type)?;
747-
Ok((name.clone(), data_type))
748-
})
749-
.collect::<Result<Vec<_>>>()?;
750-
let insert_op = match conf.insert_op() {
751-
protobuf::InsertOp::Append => InsertOp::Append,
752-
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
753-
protobuf::InsertOp::Replace => InsertOp::Replace,
754-
};
755-
let file_output_mode = match conf.file_output_mode() {
756-
protobuf::FileOutputMode::Automatic => {
757-
datafusion_datasource::file_sink_config::FileOutputMode::Automatic
758-
}
759-
protobuf::FileOutputMode::SingleFile => {
760-
datafusion_datasource::file_sink_config::FileOutputMode::SingleFile
761-
}
762-
protobuf::FileOutputMode::Directory => {
763-
datafusion_datasource::file_sink_config::FileOutputMode::Directory
764-
}
765-
};
766-
Ok(Self {
767-
original_url: String::default(),
768-
object_store_url: ObjectStoreUrl::parse(&conf.object_store_url)?,
769-
file_group,
770-
table_paths,
771-
output_schema: Arc::new(convert_required!(conf.output_schema)?),
772-
table_partition_cols,
773-
insert_op,
774-
keep_partition_by_columns: conf.keep_partition_by_columns,
775-
file_extension: conf.file_extension.clone(),
776-
file_output_mode,
777-
})
730+
FileSinkConfig::from_proto(conf)
778731
}
779732
}
780733

‎datafusion/proto/src/physical_plan/to_proto.rs‎

Lines changed: 1 addition & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -632,47 +632,6 @@ impl TryFromProto<&FileSinkConfig> for protobuf::FileSinkConfig {
632632
type Error = DataFusionError;
633633

634634
fn try_from_proto(conf: &FileSinkConfig) -> Result<Self, Self::Error> {
635-
let file_groups = conf
636-
.file_group
637-
.iter()
638-
.map(protobuf::PartitionedFile::try_from_proto)
639-
.collect::<Result<Vec<_>>>()?;
640-
let table_paths = conf
641-
.table_paths
642-
.iter()
643-
.map(ToString::to_string)
644-
.collect::<Vec<_>>();
645-
let table_partition_cols = conf
646-
.table_partition_cols
647-
.iter()
648-
.map(|(name, data_type)| {
649-
Ok(protobuf::PartitionColumn {
650-
name: name.to_owned(),
651-
arrow_type: Some(data_type.try_into()?),
652-
})
653-
})
654-
.collect::<Result<Vec<_>>>()?;
655-
let file_output_mode = match conf.file_output_mode {
656-
datafusion_datasource::file_sink_config::FileOutputMode::Automatic => {
657-
protobuf::FileOutputMode::Automatic
658-
}
659-
datafusion_datasource::file_sink_config::FileOutputMode::SingleFile => {
660-
protobuf::FileOutputMode::SingleFile
661-
}
662-
datafusion_datasource::file_sink_config::FileOutputMode::Directory => {
663-
protobuf::FileOutputMode::Directory
664-
}
665-
};
666-
Ok(Self {
667-
object_store_url: conf.object_store_url.to_string(),
668-
file_groups,
669-
table_paths,
670-
output_schema: Some(conf.output_schema.as_ref().try_into()?),
671-
table_partition_cols,
672-
keep_partition_by_columns: conf.keep_partition_by_columns,
673-
insert_op: conf.insert_op as i32,
674-
file_extension: conf.file_extension.to_string(),
675-
file_output_mode: file_output_mode.into(),
676-
})
635+
conf.to_proto()
677636
}
678637
}

0 commit comments

Comments
 (0)