Skip to content

Commit a9e19cf

Browse files
committed
fix(proto): validate file sink config decoding
Protobuf enum accessors silently replace unknown values with defaults. Rejecting malformed values prevents serialized plans from changing sink behavior and verifies compatibility decoding against the encoded config. Refs apache#23498
1 parent 9bbba48 commit a9e19cf

2 files changed

Lines changed: 107 additions & 10 deletions

File tree

‎datafusion/datasource/src/file_sink_config/proto.rs‎

Lines changed: 99 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ use datafusion_physical_expr_common::sort_expr::LexRequirement;
2929
use datafusion_physical_plan::proto::ExecutionPlanDecodeCtx;
3030
use datafusion_proto_models::protobuf;
3131

32+
use crate::ListingTableUrl;
3233
use crate::file_groups::FileGroup;
3334
use crate::file_sink_config::{FileOutputMode, FileSinkConfig};
34-
use crate::ListingTableUrl;
3535

3636
impl FileSinkConfig {
3737
/// Serialize this shared file-sink configuration without format-specific
@@ -104,12 +104,25 @@ impl FileSinkConfig {
104104
Ok((name.clone(), data_type))
105105
})
106106
.collect::<Result<Vec<_>>>()?;
107-
let insert_op = match conf.insert_op() {
107+
let insert_op = protobuf::InsertOp::try_from(conf.insert_op).map_err(|_| {
108+
internal_datafusion_err!(
109+
"Received a FileSinkConfig message with unknown InsertOp {}",
110+
conf.insert_op
111+
)
112+
})?;
113+
let insert_op = match insert_op {
108114
protobuf::InsertOp::Append => InsertOp::Append,
109115
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
110116
protobuf::InsertOp::Replace => InsertOp::Replace,
111117
};
112-
let file_output_mode = match conf.file_output_mode() {
118+
let file_output_mode = protobuf::FileOutputMode::try_from(conf.file_output_mode)
119+
.map_err(|_| {
120+
internal_datafusion_err!(
121+
"Received a FileSinkConfig message with unknown FileOutputMode {}",
122+
conf.file_output_mode
123+
)
124+
})?;
125+
let file_output_mode = match file_output_mode {
113126
protobuf::FileOutputMode::Automatic => FileOutputMode::Automatic,
114127
protobuf::FileOutputMode::SingleFile => FileOutputMode::SingleFile,
115128
protobuf::FileOutputMode::Directory => FileOutputMode::Directory,
@@ -162,3 +175,86 @@ pub fn parse_sink_sort_order(
162175
.collect::<Result<Vec<_>>>()?;
163176
Ok(LexRequirement::new(sort_exprs.into_iter().map(Into::into)))
164177
}
178+
#[cfg(test)]
179+
mod tests {
180+
use datafusion_common::DataFusionError;
181+
182+
use super::*;
183+
184+
fn valid_file_sink_config() -> protobuf::FileSinkConfig {
185+
protobuf::FileSinkConfig {
186+
object_store_url: ObjectStoreUrl::local_filesystem().to_string(),
187+
output_schema: Some(
188+
(&Schema::empty())
189+
.try_into()
190+
.expect("empty schema should serialize"),
191+
),
192+
insert_op: protobuf::InsertOp::Append.into(),
193+
file_output_mode: protobuf::FileOutputMode::Automatic.into(),
194+
..Default::default()
195+
}
196+
}
197+
198+
fn assert_decode_error(
199+
mutate: impl FnOnce(&mut protobuf::FileSinkConfig),
200+
expected: impl AsRef<str>,
201+
) {
202+
let mut conf = valid_file_sink_config();
203+
mutate(&mut conf);
204+
205+
let error =
206+
FileSinkConfig::from_proto(&conf).expect_err("invalid config should fail");
207+
match error {
208+
DataFusionError::Internal(message) => {
209+
let message = message
210+
.split_once(DataFusionError::BACK_TRACE_SEP)
211+
.map_or(message.as_str(), |(message, _)| message);
212+
assert_eq!(message, expected.as_ref());
213+
}
214+
error => panic!("expected internal error, got {error}"),
215+
}
216+
}
217+
218+
#[test]
219+
fn rejects_unknown_insert_op() {
220+
assert_decode_error(
221+
|conf| conf.insert_op = i32::MAX,
222+
format!(
223+
"Received a FileSinkConfig message with unknown InsertOp {}",
224+
i32::MAX
225+
),
226+
);
227+
}
228+
229+
#[test]
230+
fn rejects_unknown_file_output_mode() {
231+
assert_decode_error(
232+
|conf| conf.file_output_mode = i32::MAX,
233+
format!(
234+
"Received a FileSinkConfig message with unknown FileOutputMode {}",
235+
i32::MAX
236+
),
237+
);
238+
}
239+
240+
#[test]
241+
fn rejects_missing_output_schema() {
242+
assert_decode_error(
243+
|conf| conf.output_schema = None,
244+
"FileSinkConfig is missing required field 'output_schema'",
245+
);
246+
}
247+
248+
#[test]
249+
fn rejects_partition_column_without_arrow_type() {
250+
assert_decode_error(
251+
|conf| {
252+
conf.table_partition_cols.push(protobuf::PartitionColumn {
253+
name: "partition".to_string(),
254+
arrow_type: None,
255+
});
256+
},
257+
"PartitionColumn is missing required field 'arrow_type'",
258+
);
259+
}
260+
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2162,13 +2162,14 @@ fn file_sink_config_conversion_preserves_compatibility_api() -> Result<()> {
21622162
file_output_mode: FileOutputMode::Directory,
21632163
};
21642164

2165-
let direct = config.to_proto()?;
2166-
let compatibility = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167-
assert_eq!(direct, compatibility);
2168-
2169-
let direct = FileSinkConfig::from_proto(&direct)?;
2170-
let compatibility = FileSinkConfig::try_from_proto(&compatibility)?;
2171-
assert_eq!(direct.to_proto()?, compatibility.to_proto()?);
2165+
let encoded = config.to_proto()?;
2166+
let compatibility_encoded = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167+
assert_eq!(encoded, compatibility_encoded);
2168+
2169+
let decoded = FileSinkConfig::from_proto(&encoded)?;
2170+
let compatibility_decoded = FileSinkConfig::try_from_proto(&encoded)?;
2171+
assert_eq!(decoded.to_proto()?, encoded);
2172+
assert_eq!(compatibility_decoded.to_proto()?, encoded);
21722173
Ok(())
21732174
}
21742175

0 commit comments

Comments
 (0)