@@ -666,6 +666,28 @@ impl Display for ConfigNonZeroUsize {
666666 }
667667}
668668
669+ /// `ConfigField` for `Option<ConfigNonZeroUsize>`. A custom impl (rather than
670+ /// the blanket `Option<F>` one) so an invalid value is rejected without leaving
671+ /// the option in an invalid intermediate state on error.
672+ impl ConfigField for Option < ConfigNonZeroUsize > {
673+ fn visit < V : Visit > ( & self , v : & mut V , key : & str , description : & ' static str ) {
674+ match self {
675+ Some ( s) => v. some ( key, s, description) ,
676+ None => v. none ( key, description) ,
677+ }
678+ }
679+
680+ fn set ( & mut self , _key : & str , value : & str ) -> Result < ( ) > {
681+ * self = Some ( ConfigNonZeroUsize :: from_str ( value) ?) ;
682+ Ok ( ( ) )
683+ }
684+
685+ fn reset ( & mut self , _key : & str ) -> Result < ( ) > {
686+ * self = None ;
687+ Ok ( ( ) )
688+ }
689+ }
690+
669691/// A `usize` configuration value that rejects 0 and 1 when set from strings.
670692///
671693/// Use this for options whose consumer divides the value in half to size an
@@ -1236,11 +1258,13 @@ config_namespace! {
12361258 pub enabled: bool , default = false
12371259
12381260 /// Minimum chunk size in bytes. The rolling hash will not trigger a split
1239- /// until this many bytes have been accumulated. Default is 256 KiB.
1240- pub min_chunk_size: usize , default = 256 * 1024
1261+ /// until this many bytes have been accumulated. Must be greater than 0.
1262+ /// Default is 256 KiB.
1263+ pub min_chunk_size: ConfigNonZeroUsize , default = non_zero_usize_default( 256 * 1024 )
12411264
12421265 /// Maximum chunk size in bytes. A split is forced when the accumulated
1243- /// size exceeds this value. Default is 1 MiB.
1266+ /// size exceeds this value. Must be greater than `min_chunk_size`.
1267+ /// Default is 1 MiB.
12441268 pub max_chunk_size: usize , default = 1024 * 1024
12451269
12461270 /// Normalization level. Increasing this improves deduplication ratio
@@ -1335,6 +1359,72 @@ impl ConfigField for Option<MaxRowGroupBytes> {
13351359 }
13361360}
13371361
1362+ /// Parquet bloom filter false positive probability.
1363+ ///
1364+ /// Wraps an `f64` so the "strictly between 0 and 1" constraint (arrow-rs
1365+ /// panics on any other value, including NaN) is validated when the config is
1366+ /// set, rather than when the writer properties are built.
1367+ #[ derive( Clone , Copy , Debug , PartialEq ) ]
1368+ pub struct BloomFilterFpp ( f64 ) ;
1369+
1370+ impl BloomFilterFpp {
1371+ /// Creates a `BloomFilterFpp`, rejecting values outside `(0.0, 1.0)`.
1372+ pub fn try_new ( value : f64 ) -> Result < Self > {
1373+ // Written so that NaN, which fails every comparison, is rejected too
1374+ if !( value > 0.0 && value < 1.0 ) {
1375+ return Err ( DataFusionError :: Configuration ( format ! (
1376+ "bloom_filter_fpp must be between 0 and 1 (exclusive), got {value}"
1377+ ) ) ) ;
1378+ }
1379+ Ok ( Self ( value) )
1380+ }
1381+
1382+ /// Returns the configured false positive probability.
1383+ pub fn get ( & self ) -> f64 {
1384+ self . 0
1385+ }
1386+ }
1387+
1388+ impl FromStr for BloomFilterFpp {
1389+ type Err = DataFusionError ;
1390+
1391+ fn from_str ( s : & str ) -> Result < Self , Self :: Err > {
1392+ let value = s. parse :: < f64 > ( ) . map_err ( |_| {
1393+ DataFusionError :: Configuration ( format ! (
1394+ "Invalid bloom_filter_fpp: '{s}'. Expected a number between 0 and 1 (exclusive)."
1395+ ) )
1396+ } ) ?;
1397+ Self :: try_new ( value)
1398+ }
1399+ }
1400+
1401+ impl Display for BloomFilterFpp {
1402+ fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
1403+ write ! ( f, "{}" , self . 0 )
1404+ }
1405+ }
1406+
1407+ /// `ConfigField` for `Option<BloomFilterFpp>`, for the same reason as the
1408+ /// `Option<MaxRowGroupBytes>` impl above.
1409+ impl ConfigField for Option < BloomFilterFpp > {
1410+ fn visit < V : Visit > ( & self , v : & mut V , key : & str , description : & ' static str ) {
1411+ match self {
1412+ Some ( s) => v. some ( key, s, description) ,
1413+ None => v. none ( key, description) ,
1414+ }
1415+ }
1416+
1417+ fn set ( & mut self , _key : & str , value : & str ) -> Result < ( ) > {
1418+ * self = Some ( BloomFilterFpp :: from_str ( value) ?) ;
1419+ Ok ( ( ) )
1420+ }
1421+
1422+ fn reset ( & mut self , _key : & str ) -> Result < ( ) > {
1423+ * self = None ;
1424+ Ok ( ( ) )
1425+ }
1426+ }
1427+
13381428config_namespace ! {
13391429 /// Options for reading and writing parquet files
13401430 ///
@@ -1472,8 +1562,8 @@ config_namespace! {
14721562 /// (writing) Sets best effort maximum size of data page in bytes
14731563 pub data_pagesize_limit: usize , default = 1024 * 1024
14741564
1475- /// (writing) Sets write_batch_size in rows
1476- pub write_batch_size: usize , default = 1024
1565+ /// (writing) Sets write_batch_size in rows. Must be greater than 0.
1566+ pub write_batch_size: ConfigNonZeroUsize , default = non_zero_usize_default ( 1024 )
14771567
14781568 /// (writing) Sets parquet writer version
14791569 /// valid values are "1.0" and "2.0"
@@ -1512,8 +1602,8 @@ config_namespace! {
15121602 /// rows). Writing larger row groups requires more memory to write, but
15131603 /// can get better compression and be faster to read. When
15141604 /// `max_row_group_bytes` is also set, the writer flushes a row group when
1515- /// either limit is reached, whichever comes first.
1516- pub max_row_group_size: usize , default = 1024 * 1024
1605+ /// either limit is reached, whichever comes first. Must be greater than 0.
1606+ pub max_row_group_size: ConfigNonZeroUsize , default = non_zero_usize_default ( 1024 * 1024 )
15171607
15181608 /// (writing) Target maximum size of each row group in bytes. When set,
15191609 /// the writer flushes whenever either this limit or `max_row_group_size`
@@ -1528,15 +1618,17 @@ config_namespace! {
15281618 /// (writing) Sets "created by" property
15291619 pub created_by: String , default = concat!( "datafusion version " , env!( "CARGO_PKG_VERSION" ) ) . into( )
15301620
1531- /// (writing) Sets column index truncate length
1532- pub column_index_truncate_length: Option <usize >, default = Some ( 64 )
1621+ /// (writing) Sets column index truncate length. Must be greater than 0.
1622+ /// If NULL, column index values are not truncated
1623+ pub column_index_truncate_length: Option <ConfigNonZeroUsize >, default = Some ( non_zero_usize_default( 64 ) )
15331624
1534- /// (writing) Sets statistics truncate length. If NULL, uses
1535- /// default parquet writer setting
1536- pub statistics_truncate_length: Option <usize >, default = Some ( 64 )
1625+ /// (writing) Sets statistics truncate length. Must be greater than 0.
1626+ /// If NULL, uses default parquet writer setting
1627+ pub statistics_truncate_length: Option <ConfigNonZeroUsize >, default = Some ( non_zero_usize_default ( 64 ) )
15371628
1538- /// (writing) Sets best effort maximum number of rows in data page
1539- pub data_page_row_count_limit: usize , default = 20_000
1629+ /// (writing) Sets best effort maximum number of rows in data page.
1630+ /// Must be greater than 0.
1631+ pub data_page_row_count_limit: ConfigNonZeroUsize , default = non_zero_usize_default( 20_000 )
15401632
15411633 /// (writing) Sets default encoding for any column.
15421634 /// Valid values are: plain, plain_dictionary, rle,
@@ -1549,9 +1641,9 @@ config_namespace! {
15491641 /// (writing) Write bloom filters for all columns when creating parquet files
15501642 pub bloom_filter_on_write: bool , default = false
15511643
1552- /// (writing) Sets bloom filter false positive probability. If NULL, uses
1553- /// default parquet writer setting
1554- pub bloom_filter_fpp: Option <f64 >, default = None
1644+ /// (writing) Sets bloom filter false positive probability. Must be
1645+ /// between 0 and 1 (exclusive). If NULL, uses default parquet writer setting
1646+ pub bloom_filter_fpp: Option <BloomFilterFpp >, default = None
15551647
15561648 /// (writing) Sets bloom filter number of distinct values. If NULL, uses
15571649 /// default parquet writer setting
@@ -3504,9 +3596,9 @@ config_namespace_with_hashmap! {
35043596 /// default parquet options
35053597 pub statistics_enabled: Option <String >, default = None
35063598
3507- /// Sets bloom filter false positive probability for the column path. If NULL, uses
3508- /// default parquet options
3509- pub bloom_filter_fpp: Option <f64 >, default = None
3599+ /// Sets bloom filter false positive probability for the column path. Must be
3600+ /// between 0 and 1 (exclusive). If NULL, uses default parquet options
3601+ pub bloom_filter_fpp: Option <BloomFilterFpp >, default = None
35103602
35113603 /// Sets bloom filter number of distinct values. If NULL, uses
35123604 /// default parquet options
@@ -5101,7 +5193,7 @@ mod tests {
51015193 . unwrap ( ) ;
51025194 let cdc = & config. execution . parquet . content_defined_chunking ;
51035195 assert ! ( cdc. enabled) ;
5104- assert_eq ! ( cdc. min_chunk_size, 256 * 1024 ) ;
5196+ assert_eq ! ( cdc. min_chunk_size. get ( ) , 256 * 1024 ) ;
51055197 assert_eq ! ( cdc. max_chunk_size, 1024 * 1024 ) ;
51065198 assert_eq ! ( cdc. norm_level, 0 ) ;
51075199
@@ -5132,7 +5224,7 @@ mod tests {
51325224 . unwrap ( ) ;
51335225 let cdc = & config. execution . parquet . content_defined_chunking ;
51345226 assert ! ( !cdc. enabled) ;
5135- assert_eq ! ( cdc. min_chunk_size, 1024 ) ;
5227+ assert_eq ! ( cdc. min_chunk_size. get ( ) , 1024 ) ;
51365228 assert_eq ! ( cdc. max_chunk_size, 1024 * 1024 ) ;
51375229 assert_eq ! ( cdc. norm_level, 0 ) ;
51385230 }
0 commit comments