Skip to content

Commit 2a7a1e3

Browse files
authored
fix(proto): check plan integer conversions across usize boundaries (apache#24483)
## Which issue does this PR close? - Part of apache#24170. ## Rationale for this change Several logical and physical plan protobuf paths convert wire integers to `usize` with unchecked casts. On 32-bit targets, values above `u32::MAX` silently truncate. For row limits, a fetch of `1 << 32` becomes `0`, causing a plan that should return rows to return an empty result. The reverse conversion can also truncate when a 64-bit `usize` does not fit the protobuf field. ## What changes are included in this PR? - Add shared helpers for checked conversions between wire integers and `usize`, with errors identifying the plan node and field. - Apply checked conversions to limits, partition counts, batch sizes, capacities, and indexes across logical and physical plan serialization. - Preserve the documented `-1` sentinel for limits without a fetch. - Reject invalid zero batch sizes while preserving proto3 defaults for older `FilterExec` payloads. - Treat out-of-range `usize` statistics as unknown rather than truncating them. The protobuf wire format is unchanged. Follow-up work will make common protobuf option and constraint conversions fallible. ## Are these changes tested? Yes. Added unit and protobuf roundtrip coverage for checked conversions, architecture-dependent integer boundaries, oversized encode/decode values, limits without a fetch, malformed zero-column `Values` nodes, and zero batch sizes. ## Are there any user-facing changes? Out-of-range plan integers now return a clear planning error instead of silently truncating. Invalid zero batch sizes are rejected, and out-of-range statistics become unknown. There are no breaking public API or protobuf wire-format changes.
1 parent e387d95 commit 2a7a1e3

28 files changed

Lines changed: 556 additions & 180 deletions

File tree

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ pub mod proxy;
2525
pub mod string_utils;
2626

2727
use crate::assert_or_internal_err;
28-
use crate::error::{_exec_datafusion_err, _exec_err, _internal_datafusion_err};
28+
use crate::error::{
29+
_exec_datafusion_err, _exec_err, _internal_datafusion_err, _plan_datafusion_err,
30+
};
2931
use crate::{Result, ScalarValue};
3032
use arrow::array::{
3133
Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, OffsetSizeTrait,
@@ -1142,6 +1144,32 @@ pub fn combine_limit(
11421144
(combined_skip, combined_fetch)
11431145
}
11441146

1147+
/// Converts a wire integer to `usize`, rejecting out-of-range values.
1148+
/// `context` and `field` identify the value in the error message.
1149+
pub fn usize_from_wire<T>(value: T, context: &str, field: &str) -> Result<usize>
1150+
where
1151+
T: TryInto<usize> + std::fmt::Display + Copy,
1152+
{
1153+
value.try_into().map_err(|_| {
1154+
_plan_datafusion_err!(
1155+
"{context}: {field} wire value {value} is out of range for usize"
1156+
)
1157+
})
1158+
}
1159+
1160+
/// Converts a `usize` to a wire integer, rejecting out-of-range values.
1161+
pub fn usize_to_wire<T: TryFrom<usize>>(
1162+
value: usize,
1163+
context: &str,
1164+
field: &str,
1165+
) -> Result<T> {
1166+
T::try_from(value).map_err(|_| {
1167+
_plan_datafusion_err!(
1168+
"{context}: {field} value {value} is out of range for the plan wire format"
1169+
)
1170+
})
1171+
}
1172+
11451173
/// Returns the estimated number of threads available for parallel execution.
11461174
///
11471175
/// This is a wrapper around `std::thread::available_parallelism`, providing a default value
@@ -1482,6 +1510,30 @@ mod tests {
14821510
#[cfg(feature = "sql")]
14831511
use sqlparser::ast::Ident;
14841512

1513+
#[test]
1514+
fn test_usize_wire_conversions() {
1515+
assert_eq!(usize_from_wire(42_u64, "SomeExec", "fetch").unwrap(), 42);
1516+
let err = usize_from_wire(-1_i64, "SomeExec", "skip").unwrap_err();
1517+
assert_eq!(
1518+
err.strip_backtrace(),
1519+
"Error during planning: SomeExec: skip wire value -1 is out of range for usize"
1520+
);
1521+
1522+
assert_eq!(usize_to_wire::<u32>(42, "SomeExec", "fetch").unwrap(), 42);
1523+
let err = usize_to_wire::<u8>(256, "SomeExec", "fetch").unwrap_err();
1524+
assert_eq!(
1525+
err.strip_backtrace(),
1526+
"Error during planning: SomeExec: fetch value 256 is out of range for the plan wire format"
1527+
);
1528+
1529+
let max = usize_from_wire(u64::MAX, "SomeExec", "fetch");
1530+
if usize::BITS >= u64::BITS {
1531+
assert_eq!(max.unwrap(), usize::MAX);
1532+
} else {
1533+
assert!(max.is_err());
1534+
}
1535+
}
1536+
14851537
#[test]
14861538
fn test_bisect_linear_left_and_right() -> Result<()> {
14871539
let arrays: Vec<ArrayRef> = vec![

‎datafusion/datasource/src/file_scan_config/proto.rs‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ use std::sync::Arc;
3939

4040
use arrow::datatypes::Schema;
4141
use datafusion_common::parsers::CompressionTypeVariant;
42+
use datafusion_common::utils::{usize_from_wire, usize_to_wire};
4243
use datafusion_common::{DataFusionError, Result, internal_datafusion_err};
4344
use datafusion_execution::object_store::ObjectStoreUrl;
4445
use datafusion_physical_expr::projection::{ProjectionExpr, ProjectionExprs};
@@ -128,7 +129,11 @@ impl FileScanConfig {
128129
Ok(protobuf::FileScanExecConf {
129130
file_groups,
130131
statistics: Some((&self.statistics()).into()),
131-
limit: self.limit.map(|l| protobuf::ScanLimit { limit: l as u32 }),
132+
limit: self
133+
.limit
134+
.map(|limit| usize_to_wire::<u32>(limit, "FileScanConfig", "limit"))
135+
.transpose()?
136+
.map(|limit| protobuf::ScanLimit { limit }),
132137
projection: vec![],
133138
schema: Some((&schema).try_into()?),
134139
table_partition_cols: self
@@ -245,14 +250,28 @@ impl FileScanConfig {
245250
file_source
246251
};
247252

253+
let limit = conf
254+
.limit
255+
.as_ref()
256+
.map(|limit| usize_from_wire(limit.limit, "FileScanConfig", "limit"))
257+
.transpose()?;
258+
let batch_size = conf
259+
.batch_size
260+
.map(|size| usize_from_wire(size, "FileScanConfig", "batch_size"))
261+
.transpose()?;
262+
if batch_size == Some(0) {
263+
return datafusion_common::plan_err!(
264+
"FileScanConfig: batch_size must be greater than 0"
265+
);
266+
}
248267
let config_builder = FileScanConfigBuilder::new(object_store_url, file_source)
249268
.with_file_groups(file_groups)
250269
.with_constraints(constraints)
251270
.with_statistics(statistics)
252-
.with_limit(conf.limit.as_ref().map(|sl| sl.limit as usize))
271+
.with_limit(limit)
253272
.with_output_ordering(output_ordering)
254273
.with_output_partitioning(output_partitioning)
255-
.with_batch_size(conf.batch_size.map(|s| s as usize))
274+
.with_batch_size(batch_size)
256275
.with_file_compression_type(file_compression_type);
257276
Ok(config_builder.build())
258277
}

‎datafusion/datasource/src/memory.rs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,7 @@ impl DataSource for MemorySourceConfig {
280280
&self,
281281
ctx: &datafusion_physical_plan::proto::ExecutionPlanEncodeCtx<'_>,
282282
) -> Result<Option<datafusion_proto_models::protobuf::PhysicalPlanNode>> {
283+
use datafusion_common::utils::usize_to_wire;
283284
use datafusion_physical_expr_common::sort_expr::sort_exprs_try_to_proto;
284285
use datafusion_proto_models::protobuf;
285286

@@ -315,7 +316,12 @@ impl DataSource for MemorySourceConfig {
315316
projection,
316317
sort_information,
317318
show_sizes: self.show_sizes,
318-
fetch: self.fetch.map(|f| f as u32),
319+
fetch: self
320+
.fetch
321+
.map(|fetch| {
322+
usize_to_wire(fetch, "MemoryScanExecNode", "fetch")
323+
})
324+
.transpose()?,
319325
},
320326
),
321327
),
@@ -682,6 +688,7 @@ impl MemorySourceConfig {
682688
ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>,
683689
) -> Result<Arc<dyn datafusion_physical_plan::ExecutionPlan>> {
684690
use datafusion_common::internal_datafusion_err;
691+
use datafusion_common::utils::usize_from_wire;
685692
use datafusion_physical_expr_common::sort_expr::sort_exprs_try_from_proto;
686693
use datafusion_proto_models::protobuf;
687694

@@ -718,8 +725,12 @@ impl MemorySourceConfig {
718725
sort_information.extend(LexOrdering::new(sort_exprs));
719726
}
720727

728+
let fetch = scan
729+
.fetch
730+
.map(|fetch| usize_from_wire(fetch, "MemoryScanExecNode", "fetch"))
731+
.transpose()?;
721732
let source = Self::try_new(&partitions, schema, projection)?
722-
.with_limit(scan.fetch.map(|f| f as usize))
733+
.with_limit(fetch)
723734
.with_show_sizes(scan.show_sizes)
724735
.try_with_sort_information(sort_information)?;
725736

‎datafusion/expr/src/logical_plan/plan.rs‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ use datafusion_common::{
6565
FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result,
6666
ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions,
6767
aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err,
68-
internal_err, plan_err, validate_range_split_points,
68+
internal_err, plan_datafusion_err, plan_err, validate_range_split_points,
6969
};
7070
use indexmap::IndexSet;
7171
use itertools::Itertools as _;
@@ -3782,7 +3782,12 @@ impl Limit {
37823782
// `skip = NULL` is equivalent to `skip = 0`
37833783
let s = s.unwrap_or(0);
37843784
if s >= 0 {
3785-
Ok(SkipType::Literal(s as usize))
3785+
let s = usize::try_from(s).map_err(|_| {
3786+
plan_datafusion_err!(
3787+
"OFFSET value {s} cannot be represented as usize"
3788+
)
3789+
})?;
3790+
Ok(SkipType::Literal(s))
37863791
} else {
37873792
plan_err!("OFFSET must be >=0, '{}' was provided", s)
37883793
}
@@ -3800,7 +3805,12 @@ impl Limit {
38003805
Some(expr) => match *expr {
38013806
Expr::Literal(ScalarValue::Int64(Some(s)), _) => {
38023807
if s >= 0 {
3803-
Ok(FetchType::Literal(Some(s as usize)))
3808+
let s = usize::try_from(s).map_err(|_| {
3809+
plan_datafusion_err!(
3810+
"LIMIT value {s} cannot be represented as usize"
3811+
)
3812+
})?;
3813+
Ok(FetchType::Literal(Some(s)))
38043814
} else {
38053815
plan_err!("LIMIT must be >= 0, '{}' was provided", s)
38063816
}
@@ -5031,6 +5041,33 @@ mod tests {
50315041
);
50325042
}
50335043

5044+
#[test]
5045+
fn limit_literals_use_checked_usize_conversion() -> Result<()> {
5046+
let value = i64::from(u32::MAX) + 1;
5047+
let input = Arc::new(LogicalPlanBuilder::empty(false).build()?);
5048+
let limit = Limit {
5049+
skip: Some(Box::new(lit(value))),
5050+
fetch: Some(Box::new(lit(value))),
5051+
input,
5052+
};
5053+
5054+
if usize::BITS < 64 {
5055+
assert!(limit.get_skip_type().is_err());
5056+
assert!(limit.get_fetch_type().is_err());
5057+
} else {
5058+
let expected = usize::try_from(value).unwrap();
5059+
let SkipType::Literal(skip) = limit.get_skip_type()? else {
5060+
panic!("expected literal skip")
5061+
};
5062+
let FetchType::Literal(Some(fetch)) = limit.get_fetch_type()? else {
5063+
panic!("expected literal fetch")
5064+
};
5065+
assert_eq!(skip, expected);
5066+
assert_eq!(fetch, expected);
5067+
}
5068+
Ok(())
5069+
}
5070+
50345071
fn employee_schema() -> Schema {
50355072
Schema::new(vec![
50365073
Field::new("id", DataType::Int32, false),

‎datafusion/functions-table/src/generate_series.rs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,9 @@ impl GenerateSeriesTable {
291291
&self,
292292
batch_size: usize,
293293
) -> Result<Arc<RwLock<dyn LazyBatchGenerator>>> {
294+
if batch_size == 0 {
295+
return plan_err!("GenerateSeriesTable: batch_size must be greater than 0");
296+
}
294297
let generator: Arc<RwLock<dyn LazyBatchGenerator>> = match &self.args {
295298
GenSeriesArgs::ContainsNull { name } => Arc::new(RwLock::new(Empty { name })),
296299
GenSeriesArgs::Int64Args {
@@ -877,7 +880,26 @@ mod generate_series_tests {
877880
use datafusion_common::Result;
878881
use datafusion_physical_plan::memory::LazyBatchGenerator;
879882

880-
use crate::generate_series::GenericSeriesState;
883+
use crate::generate_series::{
884+
GenSeriesArgs, GenerateSeriesTable, GenericSeriesState,
885+
};
886+
887+
#[test]
888+
fn generate_series_rejects_zero_batch_size() {
889+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
890+
let table = GenerateSeriesTable::new(
891+
schema,
892+
GenSeriesArgs::Int64Args {
893+
start: 1,
894+
end: 2,
895+
step: 1,
896+
include_end: true,
897+
name: "generate_series",
898+
},
899+
);
900+
901+
assert!(table.as_generator(0).is_err());
902+
}
881903

882904
#[test]
883905
fn test_generic_series_state_reset() -> Result<()> {

0 commit comments

Comments
 (0)