Skip to content

Commit 54f5cf1

Browse files
authored
fix(proto): check integer conversions in common options and constraints (apache#24726)
## Which issue does this PR close? - Closes apache#24170. - Follow-up to apache#24483. ## Rationale for this change apache#24483 added checked integer conversions across logical and physical plan serialization. Common protobuf conversions for file-format options and constraints still used unchecked casts, allowing oversized values to silently truncate on 32-bit targets. ## What changes are included in this PR? - Apply the checked conversion helper from apache#24483 to common options and constraints. - Make affected conversions fallible, propagating errors instead of truncating or panicking. - Update conversion coverage and the 56.0.0 upgrade guide. The protobuf wire format is unchanged. ## Are these changes tested? Yes. Existing option round-trip tests pass, and tests cover the fallible public conversion contracts and malformed constraints. ## Are there any user-facing changes? Oversized protobuf values now return an error instead of silently truncating. Some public `From` conversions are now `TryFrom`, so downstream callers must handle the resulting error. This migration is documented in the 56.0.0 upgrade guide.
1 parent e1942b1 commit 54f5cf1

8 files changed

Lines changed: 302 additions & 114 deletions

File tree

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -941,7 +941,7 @@ impl CsvSink {
941941

942942
/// Encode a [`CsvFormatFactory`]'s options as their protobuf form.
943943
///
944-
/// The reverse direction is `From<&protobuf::CsvOptions> for CsvOptions` in
944+
/// The reverse direction is `TryFrom<&protobuf::CsvOptions> for CsvOptions` in
945945
/// `datafusion-proto-models`: `CsvOptions` is a `datafusion-common` type, so
946946
/// that half cannot live here.
947947
#[cfg(feature = "proto")]

‎datafusion/datasource-json/src/file_format.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ impl Decoder for JsonDecoder {
618618

619619
/// Encode a [`JsonFormatFactory`]'s options as their protobuf form.
620620
///
621-
/// The reverse direction is `From<&protobuf::JsonOptions> for JsonOptions` in
621+
/// The reverse direction is `TryFrom<&protobuf::JsonOptions> for JsonOptions` in
622622
/// `datafusion-proto-models`: `JsonOptions` is a `datafusion-common` type, so
623623
/// that half cannot live here.
624624
#[cfg(feature = "proto")]

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 170 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ use datafusion_common::{
4646
parsers::CompressionTypeVariant,
4747
plan_datafusion_err,
4848
stats::Precision,
49+
utils::usize_from_wire,
4950
};
5051

5152
#[derive(Debug)]
@@ -738,30 +739,23 @@ impl From<protobuf::IntervalUnit> for IntervalUnit {
738739
}
739740
}
740741

741-
impl From<protobuf::Constraints> for Constraints {
742-
fn from(constraints: protobuf::Constraints) -> Self {
743-
Constraints::new_unverified(
744-
constraints
745-
.constraints
746-
.into_iter()
747-
.map(|item| item.into())
748-
.collect(),
749-
)
742+
impl TryFrom<protobuf::Constraints> for Constraints {
743+
type Error = DataFusionError;
744+
745+
fn try_from(
746+
constraints: protobuf::Constraints,
747+
) -> datafusion_common::Result<Self, Self::Error> {
748+
Self::try_from(&constraints)
750749
}
751750
}
752751

753-
impl From<protobuf::Constraint> for Constraint {
754-
fn from(value: protobuf::Constraint) -> Self {
755-
match value.constraint_mode.unwrap() {
756-
protobuf::constraint::ConstraintMode::PrimaryKey(elem) => {
757-
Constraint::PrimaryKey(
758-
elem.indices.into_iter().map(|item| item as usize).collect(),
759-
)
760-
}
761-
protobuf::constraint::ConstraintMode::Unique(elem) => Constraint::Unique(
762-
elem.indices.into_iter().map(|item| item as usize).collect(),
763-
),
764-
}
752+
impl TryFrom<protobuf::Constraint> for Constraint {
753+
type Error = DataFusionError;
754+
755+
fn try_from(
756+
value: protobuf::Constraint,
757+
) -> datafusion_common::Result<Self, Self::Error> {
758+
Constraint::try_from(&value)
765759
}
766760
}
767761

@@ -884,20 +878,28 @@ impl From<protobuf::JoinSide> for JoinSide {
884878
}
885879
}
886880

887-
impl From<&protobuf::Constraint> for Constraint {
888-
fn from(value: &protobuf::Constraint) -> Self {
881+
impl TryFrom<&protobuf::Constraint> for Constraint {
882+
type Error = DataFusionError;
883+
884+
fn try_from(
885+
value: &protobuf::Constraint,
886+
) -> datafusion_common::Result<Self, Self::Error> {
887+
let decode_indices = |indices: &[u64]| {
888+
indices
889+
.iter()
890+
.map(|&index| usize_from_wire(index, "Constraint", "indices"))
891+
.collect::<datafusion_common::Result<Vec<_>>>()
892+
};
889893
match &value.constraint_mode {
890894
Some(protobuf::constraint::ConstraintMode::PrimaryKey(elem)) => {
891-
Constraint::PrimaryKey(
892-
elem.indices.iter().map(|&item| item as usize).collect(),
893-
)
895+
Ok(Constraint::PrimaryKey(decode_indices(&elem.indices)?))
894896
}
895897
Some(protobuf::constraint::ConstraintMode::Unique(elem)) => {
896-
Constraint::Unique(
897-
elem.indices.iter().map(|&item| item as usize).collect(),
898-
)
898+
Ok(Constraint::Unique(decode_indices(&elem.indices)?))
899899
}
900-
None => panic!("constraint_mode not set"),
900+
None => Err(plan_datafusion_err!(
901+
"Constraint: missing required field 'constraint_mode'"
902+
)),
901903
}
902904
}
903905
}
@@ -912,8 +914,8 @@ impl TryFrom<&protobuf::Constraints> for Constraints {
912914
constraints
913915
.constraints
914916
.iter()
915-
.map(|item| item.into())
916-
.collect(),
917+
.map(Constraint::try_from)
918+
.collect::<datafusion_common::Result<Vec<_>>>()?,
917919
))
918920
}
919921
}
@@ -1016,7 +1018,10 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions {
10161018
newlines_in_values: proto_opts.newlines_in_values.first().map(|h| *h != 0),
10171019
compression: proto_opts.compression().into(),
10181020
compression_level: proto_opts.compression_level,
1019-
schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),
1021+
schema_infer_max_rec: proto_opts
1022+
.schema_infer_max_rec
1023+
.map(|v| usize_from_wire(v, "CsvOptions", "schema_infer_max_rec"))
1024+
.transpose()?,
10201025
date_format: (!proto_opts.date_format.is_empty())
10211026
.then(|| proto_opts.date_format.clone()),
10221027
datetime_format: (!proto_opts.datetime_format.is_empty())
@@ -1052,21 +1057,28 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
10521057
fn try_from(
10531058
value: &protobuf::ParquetOptions,
10541059
) -> datafusion_common::Result<Self, Self::Error> {
1060+
let to_usize =
1061+
|value: u64, field: &str| usize_from_wire(value, "ParquetOptions", field);
10551062
Ok(ParquetOptions {
10561063
enable_page_index: value.enable_page_index,
10571064
pruning: value.pruning,
10581065
skip_metadata: value.skip_metadata,
10591066
metadata_size_hint: value
10601067
.metadata_size_hint_opt
10611068
.map(|opt| match opt {
1062-
protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => Some(v as usize),
1069+
protobuf::parquet_options::MetadataSizeHintOpt::MetadataSizeHint(v) => {
1070+
to_usize(v, "metadata_size_hint")
1071+
}
10631072
})
1064-
.unwrap_or(None),
1073+
.transpose()?,
10651074
pushdown_filters: value.pushdown_filters,
10661075
reorder_filters: value.reorder_filters,
10671076
force_filter_selections: value.force_filter_selections,
1068-
data_pagesize_limit: value.data_pagesize_limit as usize,
1069-
write_batch_size: value.write_batch_size as usize,
1077+
data_pagesize_limit: to_usize(
1078+
value.data_pagesize_limit,
1079+
"data_pagesize_limit",
1080+
)?,
1081+
write_batch_size: to_usize(value.write_batch_size, "write_batch_size")?,
10701082
writer_version: value.writer_version.parse().map_err(|e| {
10711083
DataFusionError::Internal(format!("Failed to parse writer_version: {e}"))
10721084
})?,
@@ -1075,29 +1087,39 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
10751087
}).unwrap_or(None),
10761088
dictionary_enabled: value.dictionary_enabled_opt.as_ref().map(|protobuf::parquet_options::DictionaryEnabledOpt::DictionaryEnabled(v)| *v),
10771089
// Continuing from where we left off in the TryFrom implementation
1078-
dictionary_page_size_limit: value.dictionary_page_size_limit as usize,
1090+
dictionary_page_size_limit: to_usize(
1091+
value.dictionary_page_size_limit,
1092+
"dictionary_page_size_limit",
1093+
)?,
10791094
statistics_enabled: value
10801095
.statistics_enabled_opt.clone()
10811096
.map(|opt| match opt {
10821097
protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),
10831098
})
10841099
.unwrap_or(None),
1085-
max_row_group_size: value.max_row_group_size as usize,
1086-
max_in_list_size: value.max_in_list_size as usize,
1100+
max_row_group_size: to_usize(value.max_row_group_size, "max_row_group_size")?,
1101+
max_in_list_size: to_usize(value.max_in_list_size, "max_in_list_size")?,
10871102
created_by: value.created_by.clone(),
10881103
column_index_truncate_length: value
10891104
.column_index_truncate_length_opt.as_ref()
10901105
.map(|opt| match opt {
1091-
protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => Some(*v as usize),
1106+
protobuf::parquet_options::ColumnIndexTruncateLengthOpt::ColumnIndexTruncateLength(v) => {
1107+
to_usize(*v, "column_index_truncate_length")
1108+
}
10921109
})
1093-
.unwrap_or(None),
1110+
.transpose()?,
10941111
statistics_truncate_length: value
10951112
.statistics_truncate_length_opt.as_ref()
10961113
.map(|opt| match opt {
1097-
protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => Some(*v as usize),
1114+
protobuf::parquet_options::StatisticsTruncateLengthOpt::StatisticsTruncateLength(v) => {
1115+
to_usize(*v, "statistics_truncate_length")
1116+
}
10981117
})
1099-
.unwrap_or(None),
1100-
data_page_row_count_limit: value.data_page_row_count_limit as usize,
1118+
.transpose()?,
1119+
data_page_row_count_limit: to_usize(
1120+
value.data_page_row_count_limit,
1121+
"data_page_row_count_limit",
1122+
)?,
11011123
encoding: value
11021124
.encoding_opt.clone()
11031125
.map(|opt| match opt {
@@ -1119,8 +1141,14 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
11191141
})
11201142
.unwrap_or(None),
11211143
allow_single_file_parallelism: value.allow_single_file_parallelism,
1122-
maximum_parallel_row_group_writers: value.maximum_parallel_row_group_writers as usize,
1123-
maximum_buffered_record_batches_per_stream: value.maximum_buffered_record_batches_per_stream as usize,
1144+
maximum_parallel_row_group_writers: to_usize(
1145+
value.maximum_parallel_row_group_writers,
1146+
"maximum_parallel_row_group_writers",
1147+
)?,
1148+
maximum_buffered_record_batches_per_stream: to_usize(
1149+
value.maximum_buffered_record_batches_per_stream,
1150+
"maximum_buffered_record_batches_per_stream",
1151+
)?,
11241152
schema_force_view_types: value.schema_force_view_types,
11251153
binary_as_string: value.binary_as_string,
11261154
coerce_int96: value.coerce_int96_opt.clone().map(|opt| match opt {
@@ -1131,24 +1159,34 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
11311159
}).unwrap_or(None),
11321160
skip_arrow_metadata: value.skip_arrow_metadata,
11331161
max_predicate_cache_size: value.max_predicate_cache_size_opt.map(|opt| match opt {
1134-
protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => Some(v as usize),
1135-
}).unwrap_or(None),
1136-
max_row_group_bytes: value.max_row_group_bytes_opt.and_then(|opt| match opt {
1137-
protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => MaxRowGroupBytes::try_new(v as usize).ok(),
1138-
}),
1139-
content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::from).unwrap_or_default(),
1162+
protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v) => {
1163+
to_usize(v, "max_predicate_cache_size")
1164+
}
1165+
}).transpose()?,
1166+
max_row_group_bytes: value.max_row_group_bytes_opt.map(|opt| match opt {
1167+
protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v) => {
1168+
MaxRowGroupBytes::try_new(to_usize(v, "max_row_group_bytes")?)
1169+
}
1170+
}).transpose()?,
1171+
content_defined_chunking: value.content_defined_chunking.map(ParquetCdcOptions::try_from).transpose()?.unwrap_or_default(),
11401172
})
11411173
}
11421174
}
11431175

1144-
impl From<protobuf::ParquetCdcOptions> for ParquetCdcOptions {
1145-
fn from(value: protobuf::ParquetCdcOptions) -> Self {
1146-
ParquetCdcOptions {
1176+
impl TryFrom<protobuf::ParquetCdcOptions> for ParquetCdcOptions {
1177+
type Error = DataFusionError;
1178+
1179+
fn try_from(
1180+
value: protobuf::ParquetCdcOptions,
1181+
) -> datafusion_common::Result<Self, Self::Error> {
1182+
let to_usize =
1183+
|value: u64, field: &str| usize_from_wire(value, "ParquetCdcOptions", field);
1184+
Ok(ParquetCdcOptions {
11471185
enabled: value.enabled,
1148-
min_chunk_size: value.min_chunk_size as usize,
1149-
max_chunk_size: value.max_chunk_size as usize,
1186+
min_chunk_size: to_usize(value.min_chunk_size, "min_chunk_size")?,
1187+
max_chunk_size: to_usize(value.max_chunk_size, "max_chunk_size")?,
11501188
norm_level: value.norm_level,
1151-
}
1189+
})
11521190
}
11531191
}
11541192

@@ -1214,9 +1252,9 @@ impl TryFrom<&protobuf::TableParquetOptions> for TableParquetOptions {
12141252
global: value
12151253
.global
12161254
.as_ref()
1217-
.map(|v| v.try_into())
1218-
.unwrap()
1219-
.unwrap(),
1255+
.map(ParquetOptions::try_from)
1256+
.transpose()?
1257+
.unwrap_or_default(),
12201258
column_specific_options,
12211259
..Default::default()
12221260
};
@@ -1234,7 +1272,10 @@ impl TryFrom<&protobuf::JsonOptions> for JsonOptions {
12341272
Ok(JsonOptions {
12351273
compression: compression.into(),
12361274
compression_level: proto_opts.compression_level,
1237-
schema_infer_max_rec: proto_opts.schema_infer_max_rec.map(|h| h as usize),
1275+
schema_infer_max_rec: proto_opts
1276+
.schema_infer_max_rec
1277+
.map(|v| usize_from_wire(v, "JsonOptions", "schema_infer_max_rec"))
1278+
.transpose()?,
12381279
newline_delimited: proto_opts.newline_delimited.unwrap_or(true),
12391280
})
12401281
}
@@ -1338,6 +1379,72 @@ mod tests {
13381379
MaxRowGroupBytes, ParquetCdcOptions, ParquetOptions, TableParquetOptions,
13391380
};
13401381

1382+
#[test]
1383+
fn constraint_requires_mode() {
1384+
let err = datafusion_common::Constraint::try_from(
1385+
&crate::protobuf_common::Constraint {
1386+
constraint_mode: None,
1387+
},
1388+
)
1389+
.unwrap_err();
1390+
assert!(
1391+
err.to_string()
1392+
.contains("Constraint: missing required field 'constraint_mode'")
1393+
);
1394+
}
1395+
1396+
#[test]
1397+
fn table_parquet_options_defaults_missing_global() {
1398+
let recovered = TableParquetOptions::try_from(
1399+
&crate::protobuf_common::TableParquetOptions::default(),
1400+
)
1401+
.expect("missing global should use defaults");
1402+
1403+
assert_eq!(recovered, TableParquetOptions::default());
1404+
}
1405+
1406+
#[test]
1407+
fn table_parquet_options_checks_nested_usize_conversion() {
1408+
let proto = crate::protobuf_common::TableParquetOptions {
1409+
global: Some(crate::protobuf_common::ParquetOptions {
1410+
data_pagesize_limit: u64::MAX,
1411+
writer_version: "1.0".to_string(),
1412+
..Default::default()
1413+
}),
1414+
..Default::default()
1415+
};
1416+
let recovered = TableParquetOptions::try_from(&proto);
1417+
1418+
if usize::BITS >= u64::BITS {
1419+
assert_eq!(recovered.unwrap().global.data_pagesize_limit, usize::MAX);
1420+
} else {
1421+
assert_eq!(
1422+
recovered.unwrap_err().strip_backtrace(),
1423+
"Error during planning: ParquetOptions: data_pagesize_limit wire value 18446744073709551615 is out of range for usize"
1424+
);
1425+
}
1426+
}
1427+
1428+
#[test]
1429+
fn table_parquet_options_propagates_nested_conversion_error() {
1430+
let proto = crate::protobuf_common::TableParquetOptions {
1431+
global: Some(crate::protobuf_common::ParquetOptions {
1432+
writer_version: "1.0".to_string(),
1433+
max_row_group_bytes_opt: Some(
1434+
crate::protobuf_common::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(0),
1435+
),
1436+
..Default::default()
1437+
}),
1438+
..Default::default()
1439+
};
1440+
1441+
let err = TableParquetOptions::try_from(&proto).unwrap_err();
1442+
assert_eq!(
1443+
err.strip_backtrace(),
1444+
"Invalid or Unsupported Configuration: max_row_group_bytes must be greater than 0"
1445+
);
1446+
}
1447+
13411448
fn parquet_options_proto_round_trip(opts: ParquetOptions) -> ParquetOptions {
13421449
let proto: crate::protobuf_common::ParquetOptions =
13431450
(&opts).try_into().expect("to_proto");

0 commit comments

Comments
 (0)