Skip to content

Commit 91da434

Browse files
committed
fix(proto): validate file sink config decoding
Protobuf enum accessors silently replace unknown values with defaults. Rejecting malformed values prevents serialized plans from changing sink behavior and verifies compatibility decoding against the encoded config. Refs apache#23498
1 parent 457c647 commit 91da434

2 files changed

Lines changed: 105 additions & 9 deletions

File tree

‎datafusion/datasource/src/file_sink_config/proto.rs‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,25 @@ impl FileSinkConfig {
107107
Ok((name.clone(), data_type))
108108
})
109109
.collect::<Result<Vec<_>>>()?;
110-
let insert_op = match conf.insert_op() {
110+
let insert_op = protobuf::InsertOp::try_from(conf.insert_op).map_err(|_| {
111+
internal_datafusion_err!(
112+
"Received a FileSinkConfig message with unknown InsertOp {}",
113+
conf.insert_op
114+
)
115+
})?;
116+
let insert_op = match insert_op {
111117
protobuf::InsertOp::Append => InsertOp::Append,
112118
protobuf::InsertOp::Overwrite => InsertOp::Overwrite,
113119
protobuf::InsertOp::Replace => InsertOp::Replace,
114120
};
115-
let file_output_mode = match conf.file_output_mode() {
121+
let file_output_mode = protobuf::FileOutputMode::try_from(conf.file_output_mode)
122+
.map_err(|_| {
123+
internal_datafusion_err!(
124+
"Received a FileSinkConfig message with unknown FileOutputMode {}",
125+
conf.file_output_mode
126+
)
127+
})?;
128+
let file_output_mode = match file_output_mode {
116129
protobuf::FileOutputMode::Automatic => FileOutputMode::Automatic,
117130
protobuf::FileOutputMode::SingleFile => FileOutputMode::SingleFile,
118131
protobuf::FileOutputMode::Directory => FileOutputMode::Directory,
@@ -230,3 +243,85 @@ fn partitioned_file_from_proto(
230243
}
231244
Ok(partitioned_file)
232245
}
246+
247+
#[cfg(test)]
248+
mod tests {
249+
use super::*;
250+
251+
fn valid_file_sink_config() -> protobuf::FileSinkConfig {
252+
protobuf::FileSinkConfig {
253+
object_store_url: ObjectStoreUrl::local_filesystem().to_string(),
254+
output_schema: Some(
255+
(&Schema::empty())
256+
.try_into()
257+
.expect("empty schema should serialize"),
258+
),
259+
insert_op: protobuf::InsertOp::Append.into(),
260+
file_output_mode: protobuf::FileOutputMode::Automatic.into(),
261+
..Default::default()
262+
}
263+
}
264+
265+
fn assert_decode_error(
266+
mutate: impl FnOnce(&mut protobuf::FileSinkConfig),
267+
expected: impl AsRef<str>,
268+
) {
269+
let mut conf = valid_file_sink_config();
270+
mutate(&mut conf);
271+
272+
let error =
273+
FileSinkConfig::from_proto(&conf).expect_err("invalid config should fail");
274+
match error {
275+
DataFusionError::Internal(message) => {
276+
let message = message
277+
.split_once(DataFusionError::BACK_TRACE_SEP)
278+
.map_or(message.as_str(), |(message, _)| message);
279+
assert_eq!(message, expected.as_ref());
280+
}
281+
error => panic!("expected internal error, got {error}"),
282+
}
283+
}
284+
285+
#[test]
286+
fn rejects_unknown_insert_op() {
287+
assert_decode_error(
288+
|conf| conf.insert_op = i32::MAX,
289+
format!(
290+
"Received a FileSinkConfig message with unknown InsertOp {}",
291+
i32::MAX
292+
),
293+
);
294+
}
295+
296+
#[test]
297+
fn rejects_unknown_file_output_mode() {
298+
assert_decode_error(
299+
|conf| conf.file_output_mode = i32::MAX,
300+
format!(
301+
"Received a FileSinkConfig message with unknown FileOutputMode {}",
302+
i32::MAX
303+
),
304+
);
305+
}
306+
307+
#[test]
308+
fn rejects_missing_output_schema() {
309+
assert_decode_error(
310+
|conf| conf.output_schema = None,
311+
"FileSinkConfig is missing required field 'output_schema'",
312+
);
313+
}
314+
315+
#[test]
316+
fn rejects_partition_column_without_arrow_type() {
317+
assert_decode_error(
318+
|conf| {
319+
conf.table_partition_cols.push(protobuf::PartitionColumn {
320+
name: "partition".to_string(),
321+
arrow_type: None,
322+
});
323+
},
324+
"PartitionColumn is missing required field 'arrow_type'",
325+
);
326+
}
327+
}

‎datafusion/proto/tests/cases/roundtrip_physical_plan.rs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2162,13 +2162,14 @@ fn file_sink_config_conversion_preserves_compatibility_api() -> Result<()> {
21622162
file_output_mode: FileOutputMode::Directory,
21632163
};
21642164

2165-
let direct = config.to_proto()?;
2166-
let compatibility = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167-
assert_eq!(direct, compatibility);
2168-
2169-
let direct = FileSinkConfig::from_proto(&direct)?;
2170-
let compatibility = FileSinkConfig::try_from_proto(&compatibility)?;
2171-
assert_eq!(direct.to_proto()?, compatibility.to_proto()?);
2165+
let encoded = config.to_proto()?;
2166+
let compatibility_encoded = protobuf::FileSinkConfig::try_from_proto(&config)?;
2167+
assert_eq!(encoded, compatibility_encoded);
2168+
2169+
let decoded = FileSinkConfig::from_proto(&encoded)?;
2170+
let compatibility_decoded = FileSinkConfig::try_from_proto(&encoded)?;
2171+
assert_eq!(decoded.to_proto()?, encoded);
2172+
assert_eq!(compatibility_decoded.to_proto()?, encoded);
21722173
Ok(())
21732174
}
21742175

0 commit comments

Comments
 (0)