Skip to content

Commit 7e19cdb

Browse files
committed
fix: reject invalid parquet writer options when they are set
Parquet writer options that the parquet crate cannot accept (a zero write_batch_size, max_row_group_size, data_page_row_count_limit, truncate length or min_chunk_size, or a bloom_filter_fpp outside (0, 1)) were accepted by SET and then panicked or hung on the next write. They are now rejected with a configuration error when set. max_chunk_size > min_chunk_size compares two options, so it is checked when the writer properties are built. Protobuf decoding applies the same checks. A wire value of 0 for a non-optional field is read as absent and replaced by the default. Closes apache#24904
1 parent a9129e7 commit 7e19cdb

13 files changed

Lines changed: 537 additions & 124 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 114 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -666,6 +666,28 @@ impl Display for ConfigNonZeroUsize {
666666
}
667667
}
668668

669+
/// `ConfigField` for `Option<ConfigNonZeroUsize>`. A custom impl (rather than
670+
/// the blanket `Option<F>` one) so an invalid value is rejected without leaving
671+
/// the option in an invalid intermediate state on error.
672+
impl ConfigField for Option<ConfigNonZeroUsize> {
673+
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
674+
match self {
675+
Some(s) => v.some(key, s, description),
676+
None => v.none(key, description),
677+
}
678+
}
679+
680+
fn set(&mut self, _key: &str, value: &str) -> Result<()> {
681+
*self = Some(ConfigNonZeroUsize::from_str(value)?);
682+
Ok(())
683+
}
684+
685+
fn reset(&mut self, _key: &str) -> Result<()> {
686+
*self = None;
687+
Ok(())
688+
}
689+
}
690+
669691
/// A `usize` configuration value that rejects 0 and 1 when set from strings.
670692
///
671693
/// Use this for options whose consumer divides the value in half to size an
@@ -1236,11 +1258,13 @@ config_namespace! {
12361258
pub enabled: bool, default = false
12371259

12381260
/// Minimum chunk size in bytes. The rolling hash will not trigger a split
1239-
/// until this many bytes have been accumulated. Default is 256 KiB.
1240-
pub min_chunk_size: usize, default = 256 * 1024
1261+
/// until this many bytes have been accumulated. Must be greater than 0.
1262+
/// Default is 256 KiB.
1263+
pub min_chunk_size: ConfigNonZeroUsize, default = non_zero_usize_default(256 * 1024)
12411264

12421265
/// Maximum chunk size in bytes. A split is forced when the accumulated
1243-
/// size exceeds this value. Default is 1 MiB.
1266+
/// size exceeds this value. Must be greater than `min_chunk_size`.
1267+
/// Default is 1 MiB.
12441268
pub max_chunk_size: usize, default = 1024 * 1024
12451269

12461270
/// Normalization level. Increasing this improves deduplication ratio
@@ -1335,6 +1359,72 @@ impl ConfigField for Option<MaxRowGroupBytes> {
13351359
}
13361360
}
13371361

1362+
/// Parquet bloom filter false positive probability.
1363+
///
1364+
/// Wraps an `f64` so the "strictly between 0 and 1" constraint (arrow-rs
1365+
/// panics on any other value, including NaN) is validated when the config is
1366+
/// set, rather than when the writer properties are built.
1367+
#[derive(Clone, Copy, Debug, PartialEq)]
1368+
pub struct BloomFilterFpp(f64);
1369+
1370+
impl BloomFilterFpp {
1371+
/// Creates a `BloomFilterFpp`, rejecting values outside `(0.0, 1.0)`.
1372+
pub fn try_new(value: f64) -> Result<Self> {
1373+
// Written so that NaN, which fails every comparison, is rejected too
1374+
if !(value > 0.0 && value < 1.0) {
1375+
return Err(DataFusionError::Configuration(format!(
1376+
"bloom_filter_fpp must be between 0 and 1 (exclusive), got {value}"
1377+
)));
1378+
}
1379+
Ok(Self(value))
1380+
}
1381+
1382+
/// Returns the configured false positive probability.
1383+
pub fn get(&self) -> f64 {
1384+
self.0
1385+
}
1386+
}
1387+
1388+
impl FromStr for BloomFilterFpp {
1389+
type Err = DataFusionError;
1390+
1391+
fn from_str(s: &str) -> Result<Self, Self::Err> {
1392+
let value = s.parse::<f64>().map_err(|_| {
1393+
DataFusionError::Configuration(format!(
1394+
"Invalid bloom_filter_fpp: '{s}'. Expected a number between 0 and 1 (exclusive)."
1395+
))
1396+
})?;
1397+
Self::try_new(value)
1398+
}
1399+
}
1400+
1401+
impl Display for BloomFilterFpp {
1402+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1403+
write!(f, "{}", self.0)
1404+
}
1405+
}
1406+
1407+
/// `ConfigField` for `Option<BloomFilterFpp>`, for the same reason as the
1408+
/// `Option<MaxRowGroupBytes>` impl above.
1409+
impl ConfigField for Option<BloomFilterFpp> {
1410+
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
1411+
match self {
1412+
Some(s) => v.some(key, s, description),
1413+
None => v.none(key, description),
1414+
}
1415+
}
1416+
1417+
fn set(&mut self, _key: &str, value: &str) -> Result<()> {
1418+
*self = Some(BloomFilterFpp::from_str(value)?);
1419+
Ok(())
1420+
}
1421+
1422+
fn reset(&mut self, _key: &str) -> Result<()> {
1423+
*self = None;
1424+
Ok(())
1425+
}
1426+
}
1427+
13381428
config_namespace! {
13391429
/// Options for reading and writing parquet files
13401430
///
@@ -1472,8 +1562,8 @@ config_namespace! {
14721562
/// (writing) Sets best effort maximum size of data page in bytes
14731563
pub data_pagesize_limit: usize, default = 1024 * 1024
14741564

1475-
/// (writing) Sets write_batch_size in rows
1476-
pub write_batch_size: usize, default = 1024
1565+
/// (writing) Sets write_batch_size in rows. Must be greater than 0.
1566+
pub write_batch_size: ConfigNonZeroUsize, default = non_zero_usize_default(1024)
14771567

14781568
/// (writing) Sets parquet writer version
14791569
/// valid values are "1.0" and "2.0"
@@ -1512,8 +1602,8 @@ config_namespace! {
15121602
/// rows). Writing larger row groups requires more memory to write, but
15131603
/// can get better compression and be faster to read. When
15141604
/// `max_row_group_bytes` is also set, the writer flushes a row group when
1515-
/// either limit is reached, whichever comes first.
1516-
pub max_row_group_size: usize, default = 1024 * 1024
1605+
/// either limit is reached, whichever comes first. Must be greater than 0.
1606+
pub max_row_group_size: ConfigNonZeroUsize, default = non_zero_usize_default(1024 * 1024)
15171607

15181608
/// (writing) Target maximum size of each row group in bytes. When set,
15191609
/// the writer flushes whenever either this limit or `max_row_group_size`
@@ -1528,15 +1618,17 @@ config_namespace! {
15281618
/// (writing) Sets "created by" property
15291619
pub created_by: String, default = concat!("datafusion version ", env!("CARGO_PKG_VERSION")).into()
15301620

1531-
/// (writing) Sets column index truncate length
1532-
pub column_index_truncate_length: Option<usize>, default = Some(64)
1621+
/// (writing) Sets column index truncate length. Must be greater than 0.
1622+
/// If NULL, column index values are not truncated
1623+
pub column_index_truncate_length: Option<ConfigNonZeroUsize>, default = Some(non_zero_usize_default(64))
15331624

1534-
/// (writing) Sets statistics truncate length. If NULL, uses
1535-
/// default parquet writer setting
1536-
pub statistics_truncate_length: Option<usize>, default = Some(64)
1625+
/// (writing) Sets statistics truncate length. Must be greater than 0.
1626+
/// If NULL, uses default parquet writer setting
1627+
pub statistics_truncate_length: Option<ConfigNonZeroUsize>, default = Some(non_zero_usize_default(64))
15371628

1538-
/// (writing) Sets best effort maximum number of rows in data page
1539-
pub data_page_row_count_limit: usize, default = 20_000
1629+
/// (writing) Sets best effort maximum number of rows in data page.
1630+
/// Must be greater than 0.
1631+
pub data_page_row_count_limit: ConfigNonZeroUsize, default = non_zero_usize_default(20_000)
15401632

15411633
/// (writing) Sets default encoding for any column.
15421634
/// Valid values are: plain, plain_dictionary, rle,
@@ -1549,9 +1641,9 @@ config_namespace! {
15491641
/// (writing) Write bloom filters for all columns when creating parquet files
15501642
pub bloom_filter_on_write: bool, default = false
15511643

1552-
/// (writing) Sets bloom filter false positive probability. If NULL, uses
1553-
/// default parquet writer setting
1554-
pub bloom_filter_fpp: Option<f64>, default = None
1644+
/// (writing) Sets bloom filter false positive probability. Must be
1645+
/// between 0 and 1 (exclusive). If NULL, uses default parquet writer setting
1646+
pub bloom_filter_fpp: Option<BloomFilterFpp>, default = None
15551647

15561648
/// (writing) Sets bloom filter number of distinct values. If NULL, uses
15571649
/// default parquet writer setting
@@ -3504,9 +3596,9 @@ config_namespace_with_hashmap! {
35043596
/// default parquet options
35053597
pub statistics_enabled: Option<String>, default = None
35063598

3507-
/// Sets bloom filter false positive probability for the column path. If NULL, uses
3508-
/// default parquet options
3509-
pub bloom_filter_fpp: Option<f64>, default = None
3599+
/// Sets bloom filter false positive probability for the column path. Must be
3600+
/// between 0 and 1 (exclusive). If NULL, uses default parquet options
3601+
pub bloom_filter_fpp: Option<BloomFilterFpp>, default = None
35103602

35113603
/// Sets bloom filter number of distinct values. If NULL, uses
35123604
/// default parquet options
@@ -5101,7 +5193,7 @@ mod tests {
51015193
.unwrap();
51025194
let cdc = &config.execution.parquet.content_defined_chunking;
51035195
assert!(cdc.enabled);
5104-
assert_eq!(cdc.min_chunk_size, 256 * 1024);
5196+
assert_eq!(cdc.min_chunk_size.get(), 256 * 1024);
51055197
assert_eq!(cdc.max_chunk_size, 1024 * 1024);
51065198
assert_eq!(cdc.norm_level, 0);
51075199

@@ -5132,7 +5224,7 @@ mod tests {
51325224
.unwrap();
51335225
let cdc = &config.execution.parquet.content_defined_chunking;
51345226
assert!(!cdc.enabled);
5135-
assert_eq!(cdc.min_chunk_size, 1024);
5227+
assert_eq!(cdc.min_chunk_size.get(), 1024);
51365228
assert_eq!(cdc.max_chunk_size, 1024 * 1024);
51375229
assert_eq!(cdc.norm_level, 0);
51385230
}

0 commit comments

Comments
 (0)