Skip to content

Commit 5ebeba3

Browse files
committed
fix: validate parquet.coerce_int96 when it is set
datafusion.execution.parquet.coerce_int96 was a plain string, so a bad value was accepted by SET and only failed on the first Parquet scan. Store it as a typed DFTimeUnit that is parsed up front, following the shape used for statistics_enabled in apache#24642, and drop the string parser in datasource-parquet along with the unwrap() that called it. Part of apache#17498
1 parent 0d79252 commit 5ebeba3

12 files changed

Lines changed: 279 additions & 61 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 72 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ use crate::encryption::{FileDecryptionProperties, FileEncryptionProperties};
2424
use crate::error::{_config_datafusion_err, _config_err};
2525
use crate::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType};
2626
use crate::parquet_config::{
27-
DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion,
27+
DFParquetCompression, DFParquetStatistics, DFParquetWriterVersion, DFTimeUnit,
2828
RowGroupRangeAssignment,
2929
};
3030
use crate::parsers::{CompressionTypeVariant, CsvQuoteStyle};
@@ -1403,13 +1403,13 @@ config_namespace! {
14031403
/// reading them as binary and then casting to string.
14041404
pub binary_as_string: bool, default = false
14051405

1406-
/// (reading) If true, parquet reader will read columns of
1407-
/// physical type int96 as originating from a different resolution
1408-
/// than nanosecond. This is useful for reading data from systems like Spark
1409-
/// which stores microsecond resolution timestamps in an int96 allowing it
1410-
/// to write values with a larger date range than 64-bit timestamps with
1411-
/// nanosecond resolution.
1412-
pub coerce_int96: Option<String>, transform = str::to_lowercase, default = None
1406+
/// (reading) If set, parquet reader will read columns of
1407+
/// physical type int96 as originating from this resolution instead of
1408+
/// nanosecond. Valid values are: ns, us, ms, and s. This is useful for
1409+
/// reading data from systems like Spark which store microsecond
1410+
/// resolution timestamps in an int96, allowing them to write values with
1411+
/// a larger date range than 64-bit timestamps with nanosecond resolution.
1412+
pub coerce_int96: Option<DFTimeUnit>, default = None
14131413

14141414
/// (reading) Optional timezone applied to INT96 columns when `coerce_int96`
14151415
/// is set. When `Some`, INT96 columns coerce to
@@ -4937,6 +4937,70 @@ mod tests {
49374937
assert_eq!(scalar, DFParquetCompression::Lz4);
49384938
}
49394939

4940+
#[cfg(feature = "parquet")]
4941+
#[test]
4942+
fn test_parquet_coerce_int96_validation() {
4943+
use crate::{config::ConfigOptions, parquet_config::DFTimeUnit};
4944+
4945+
let mut config = ConfigOptions::default();
4946+
assert_eq!(config.execution.parquet.coerce_int96, None);
4947+
4948+
for (value, expected) in [
4949+
("ns", DFTimeUnit::Nanosecond),
4950+
("US", DFTimeUnit::Microsecond),
4951+
("ms", DFTimeUnit::Millisecond),
4952+
("s", DFTimeUnit::Second),
4953+
] {
4954+
config
4955+
.set("datafusion.execution.parquet.coerce_int96", value)
4956+
.unwrap();
4957+
assert_eq!(config.execution.parquet.coerce_int96, Some(expected));
4958+
}
4959+
4960+
let err = config
4961+
.set("datafusion.execution.parquet.coerce_int96", "nanos")
4962+
.unwrap_err();
4963+
assert_contains!(
4964+
err.to_string(),
4965+
"Invalid parquet coerce_int96 setting: nanos. Expected one of: ns, us, ms, s"
4966+
);
4967+
// A rejected value leaves the previous one in place.
4968+
assert_eq!(
4969+
config.execution.parquet.coerce_int96,
4970+
Some(DFTimeUnit::Second)
4971+
);
4972+
4973+
// An invalid update must leave an unset option unset rather than
4974+
// inserting a default.
4975+
config.execution.parquet.coerce_int96 = None;
4976+
assert!(
4977+
config
4978+
.set("datafusion.execution.parquet.coerce_int96", "nanos")
4979+
.is_err()
4980+
);
4981+
assert_eq!(config.execution.parquet.coerce_int96, None);
4982+
4983+
config.execution.parquet.coerce_int96 = Some(DFTimeUnit::Microsecond);
4984+
assert!(
4985+
config
4986+
.set("datafusion.execution.parquet.coerce_int96.typo", "ns")
4987+
.is_err()
4988+
);
4989+
assert!(
4990+
config
4991+
.reset("datafusion.execution.parquet.coerce_int96.typo")
4992+
.is_err()
4993+
);
4994+
assert_eq!(
4995+
config.execution.parquet.coerce_int96,
4996+
Some(DFTimeUnit::Microsecond)
4997+
);
4998+
4999+
let mut scalar = DFTimeUnit::Microsecond;
5000+
assert!(ConfigField::set(&mut scalar, "typo", "ns").is_err());
5001+
assert_eq!(scalar, DFTimeUnit::Microsecond);
5002+
}
5003+
49405004
#[cfg(feature = "parquet")]
49415005
#[test]
49425006
fn set_cdc_enabled_flag() {

‎datafusion/common/src/parquet_config.rs‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,110 @@ impl From<DFParquetCompression> for parquet::basic::Compression {
481481
}
482482
}
483483

484+
/// Time unit for reading Parquet INT96 timestamps
485+
///
486+
/// This enum validates `coerce_int96` at configuration time, so only `ns`,
487+
/// `us`, `ms` or `s` can be set via `SET` commands or deserialization.
488+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
489+
pub enum DFTimeUnit {
490+
Nanosecond,
491+
Microsecond,
492+
Millisecond,
493+
Second,
494+
}
495+
496+
impl FromStr for DFTimeUnit {
497+
type Err = DataFusionError;
498+
499+
fn from_str(s: &str) -> Result<Self, Self::Err> {
500+
match s.to_lowercase().as_str() {
501+
"ns" => Ok(Self::Nanosecond),
502+
"us" => Ok(Self::Microsecond),
503+
"ms" => Ok(Self::Millisecond),
504+
"s" => Ok(Self::Second),
505+
other => Err(DataFusionError::Configuration(format!(
506+
"Invalid parquet coerce_int96 setting: {other}. Expected one of: ns, us, ms, s"
507+
))),
508+
}
509+
}
510+
}
511+
512+
impl Display for DFTimeUnit {
513+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
514+
let s = match self {
515+
Self::Nanosecond => "ns",
516+
Self::Microsecond => "us",
517+
Self::Millisecond => "ms",
518+
Self::Second => "s",
519+
};
520+
f.write_str(s)
521+
}
522+
}
523+
524+
impl ConfigField for DFTimeUnit {
525+
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
526+
v.some(key, self, description)
527+
}
528+
529+
fn set(&mut self, key: &str, value: &str) -> Result<()> {
530+
if !key.is_empty() {
531+
return crate::error::_config_err!(
532+
"Config field parquet.coerce_int96 is a scalar DFTimeUnit and does not have nested field \"{}\"",
533+
key
534+
);
535+
}
536+
537+
*self = Self::from_str(value)?;
538+
Ok(())
539+
}
540+
}
541+
542+
/// `ConfigField` for `Option<DFTimeUnit>` parses before assigning so an
543+
/// invalid value does not turn an unset option into the default.
544+
impl ConfigField for Option<DFTimeUnit> {
545+
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
546+
match self {
547+
Some(time_unit) => time_unit.visit(v, key, description),
548+
None => v.none(key, description),
549+
}
550+
}
551+
552+
fn set(&mut self, key: &str, value: &str) -> Result<()> {
553+
if !key.is_empty() {
554+
return crate::error::_config_err!(
555+
"Config field parquet.coerce_int96 is a scalar Option<DFTimeUnit> and does not have nested field \"{}\"",
556+
key
557+
);
558+
}
559+
560+
*self = Some(DFTimeUnit::from_str(value)?);
561+
Ok(())
562+
}
563+
564+
fn reset(&mut self, key: &str) -> Result<()> {
565+
if key.is_empty() {
566+
*self = None;
567+
Ok(())
568+
} else {
569+
crate::error::_config_err!(
570+
"Config field parquet.coerce_int96 is a scalar Option<DFTimeUnit> and does not have nested field \"{}\"",
571+
key
572+
)
573+
}
574+
}
575+
}
576+
577+
impl From<DFTimeUnit> for arrow::datatypes::TimeUnit {
578+
fn from(value: DFTimeUnit) -> Self {
579+
match value {
580+
DFTimeUnit::Nanosecond => Self::Nanosecond,
581+
DFTimeUnit::Microsecond => Self::Microsecond,
582+
DFTimeUnit::Millisecond => Self::Millisecond,
583+
DFTimeUnit::Second => Self::Second,
584+
}
585+
}
586+
}
587+
484588
#[cfg(test)]
485589
mod tests {
486590
use super::*;
@@ -566,4 +670,25 @@ mod tests {
566670
}
567671
assert!(ZstdLevel::try_new(*ZSTD_LEVELS.end() as i32 + 1).is_err());
568672
}
673+
674+
#[test]
675+
fn time_unit_parses_and_displays() {
676+
for (input, expected, displayed) in [
677+
("ns", DFTimeUnit::Nanosecond, "ns"),
678+
("US", DFTimeUnit::Microsecond, "us"),
679+
("ms", DFTimeUnit::Millisecond, "ms"),
680+
("s", DFTimeUnit::Second, "s"),
681+
] {
682+
let parsed: DFTimeUnit = input.parse().unwrap();
683+
assert_eq!(parsed, expected, "input: {input}");
684+
assert_eq!(parsed.to_string(), displayed);
685+
}
686+
for input in ["nanos", "", "ns "] {
687+
let err = input.parse::<DFTimeUnit>().unwrap_err().to_string();
688+
assert!(
689+
err.contains("Invalid parquet coerce_int96 setting"),
690+
"input {input:?}: {err}"
691+
);
692+
}
693+
}
569694
}

‎datafusion/core/src/datasource/physical_plan/parquet.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ mod tests {
4747
use arrow_schema::{SchemaRef, TimeUnit};
4848
use bytes::{BufMut, BytesMut};
4949
use datafusion_common::config::TableParquetOptions;
50+
use datafusion_common::parquet_config::DFTimeUnit;
5051
use datafusion_common::test_util::{batches_to_sort_string, batches_to_string};
5152
use datafusion_common::{Result, ScalarValue, assert_contains};
5253
use datafusion_datasource::file_format::FileFormat;
@@ -1355,7 +1356,7 @@ mod tests {
13551356
])),
13561357
),
13571358
(
1358-
Some("ns".to_string()),
1359+
Some(DFTimeUnit::Nanosecond),
13591360
Arc::new(Int64Array::from(vec![
13601361
Some(1704141296123456000),
13611362
Some(1704070800000000000),
@@ -1366,7 +1367,7 @@ mod tests {
13661367
])),
13671368
),
13681369
(
1369-
Some("us".to_string()),
1370+
Some(DFTimeUnit::Microsecond),
13701371
Arc::new(Int64Array::from(vec![
13711372
Some(1704141296123456),
13721373
Some(1704070800000000),
@@ -1381,7 +1382,7 @@ mod tests {
13811382
for (time_unit, expected) in time_units_and_expected {
13821383
let parquet_exec = scan_format(
13831384
&state,
1384-
&ParquetFormat::default().with_coerce_int96(time_unit.clone()),
1385+
&ParquetFormat::default().with_coerce_int96(time_unit),
13851386
Some(schema.clone()),
13861387
&testdata,
13871388
filename,
@@ -1430,7 +1431,7 @@ mod tests {
14301431

14311432
let parquet_exec = scan_format(
14321433
&state,
1433-
&ParquetFormat::default().with_coerce_int96(Some("us".to_string())),
1434+
&ParquetFormat::default().with_coerce_int96(Some(DFTimeUnit::Microsecond)),
14341435
None,
14351436
testdata,
14361437
filename,

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ use datafusion_datasource::file_format::{
4545
use datafusion_common::Statistics;
4646
use datafusion_common::config::{ConfigField, ConfigFileType, TableParquetOptions};
4747
use datafusion_common::encryption::FileDecryptionProperties;
48+
use datafusion_common::parquet_config::DFTimeUnit;
4849
use datafusion_common::parsers::CompressionTypeVariant;
4950
use datafusion_common::{
5051
DEFAULT_PARQUET_EXTENSION, DataFusionError, GetExt, Result, internal_datafusion_err,
@@ -61,9 +62,7 @@ use datafusion_session::Session;
6162

6263
use crate::metadata::{DFParquetMetadata, lex_ordering_to_sorting_columns};
6364
use crate::reader::CachedParquetFileReaderFactory;
64-
use crate::source::{
65-
ParquetSource, parse_coerce_int96_string, parse_coerce_int96_tz_string,
66-
};
65+
use crate::source::{ParquetSource, parse_coerce_int96_tz_string};
6766
use async_trait::async_trait;
6867
use bytes::Bytes;
6968
use datafusion_datasource::source::DataSourceExec;
@@ -242,14 +241,14 @@ impl ParquetFormat {
242241
/// Get [`coerce_int96`]
243242
///
244243
/// [`coerce_int96`]: datafusion_common::config::ParquetOptions::coerce_int96
245-
pub fn coerce_int96(&self) -> Option<String> {
246-
self.options.global.coerce_int96.clone()
244+
pub fn coerce_int96(&self) -> Option<DFTimeUnit> {
245+
self.options.global.coerce_int96
247246
}
248247

249248
/// Set [`coerce_int96`]
250249
///
251250
/// [`coerce_int96`]: datafusion_common::config::ParquetOptions::coerce_int96
252-
pub fn with_coerce_int96(mut self, time_unit: Option<String>) -> Self {
251+
pub fn with_coerce_int96(mut self, time_unit: Option<DFTimeUnit>) -> Self {
253252
self.options.global.coerce_int96 = time_unit;
254253
self
255254
}
@@ -361,10 +360,7 @@ impl FileFormat for ParquetFormat {
361360
store: &Arc<dyn ObjectStore>,
362361
objects: &[ObjectMeta],
363362
) -> Result<SchemaRef> {
364-
let coerce_int96 = match self.coerce_int96() {
365-
Some(time_unit) => Some(parse_coerce_int96_string(time_unit.as_str())?),
366-
None => None,
367-
};
363+
let coerce_int96 = self.coerce_int96().map(arrow::datatypes::TimeUnit::from);
368364
let coerce_int96_tz = self
369365
.options
370366
.global
@@ -762,8 +758,8 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
762758
write_row_group_number_distinct_values: global_options
763759
.global
764760
.write_row_group_number_distinct_values,
765-
coerce_int96_opt: global_options.global.coerce_int96.map(|compression| {
766-
parquet_options::CoerceInt96Opt::CoerceInt96(compression)
761+
coerce_int96_opt: global_options.global.coerce_int96.map(|time_unit| {
762+
parquet_options::CoerceInt96Opt::CoerceInt96(time_unit.to_string())
767763
}),
768764
coerce_int96_tz_opt: global_options.global.coerce_int96_tz.map(|tz| {
769765
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz)

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -537,24 +537,6 @@ impl ParquetSource {
537537
}
538538
}
539539

540-
/// Parses datafusion.common.config.ParquetOptions.coerce_int96 String to a arrow_schema.datatype.TimeUnit
541-
pub(crate) fn parse_coerce_int96_string(
542-
str_setting: &str,
543-
) -> datafusion_common::Result<TimeUnit> {
544-
let str_setting_lower: &str = &str_setting.to_lowercase();
545-
546-
match str_setting_lower {
547-
"ns" => Ok(TimeUnit::Nanosecond),
548-
"us" => Ok(TimeUnit::Microsecond),
549-
"ms" => Ok(TimeUnit::Millisecond),
550-
"s" => Ok(TimeUnit::Second),
551-
_ => Err(DataFusionError::Configuration(format!(
552-
"Unknown or unsupported parquet coerce_int96: \
553-
{str_setting}. Valid values are: ns, us, ms, and s."
554-
))),
555-
}
556-
}
557-
558540
/// Validates that `tz` is a parseable IANA timezone and returns it as an
559541
/// `Arc<str>` for use in `Timestamp(_, Some(tz))` types.
560542
pub(crate) fn parse_coerce_int96_tz_string(
@@ -617,8 +599,7 @@ impl FileSource for ParquetSource {
617599
.table_parquet_options
618600
.global
619601
.coerce_int96
620-
.as_ref()
621-
.map(|time_unit| parse_coerce_int96_string(time_unit.as_str()).unwrap());
602+
.map(TimeUnit::from);
622603
let coerce_int96_tz = self
623604
.table_parquet_options
624605
.global

‎datafusion/ffi/src/session/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,8 @@ mod tests {
922922
table_options.json.schema_infer_max_rec = Some(10);
923923
#[cfg(feature = "parquet")]
924924
{
925-
table_options.parquet.global.coerce_int96 = Some("123456789".into());
925+
table_options.parquet.global.coerce_int96 =
926+
Some(datafusion_common::parquet_config::DFTimeUnit::Microsecond);
926927
}
927928
table_options.current_format = Some(ConfigFileType::JSON);
928929

0 commit comments

Comments
 (0)