Skip to content

Commit d89a85d

Browse files
authored
fix: validate parquet.coerce_int96_tz when it is set. (#26420)
## Which issue does this PR close? - Closes #26362. ## Rationale for this change `SET datafusion.execution.parquet.coerce_int96_tz = 'Not/A_Timezone'` succeeds today, and the error only shows up on the next Parquet read. After #25628 the unit option next to it is checked when it is set, so this does the same for the timezone. ## What changes are included in this PR? `ParquetOptions::coerce_int96_tz` is now `Option<ConfigTimeZone>`, the type `execution.time_zone` already uses, so it accepts exactly what it accepted before. `parse_coerce_int96_tz_string` is removed, and both protobuf decoders parse the value. An invalid value is rejected even when `coerce_int96` is unset, as the issue asked. ## What is the testing strategy for this PR? - `set_variable.slt`: SET/SHOW keeps the spelling (`+08`), a bad value is rejected and leaves the old one, RESET goes back to NULL, and bad table options fail at CREATE with an inferred and an explicit schema. These fail on main. - `test_parquet_coerce_int96_tz_validation` in `config.rs`. - A rejection test for each protobuf decoder, and the existing round-trip tests use the typed value. - The existing timezone read tests in `parquet.slt` pass unchanged. ## Are there any user-facing changes? Yes. Invalid timezones are rejected when set. The public field type changes, so this needs `api change`. I added a note to the 56.0.0 upgrade guide.
1 parent a7c7c08 commit d89a85d

12 files changed

Lines changed: 346 additions & 37 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1395,8 +1395,8 @@ config_namespace! {
13951395
/// values as UTC-adjusted instants, so callers that need the resulting
13961396
/// Arrow type to be timezone-aware (e.g. for Spark `TimestampType`
13971397
/// semantics) should set this to `"UTC"`. No effect when `coerce_int96`
1398-
/// is `None`.
1399-
pub coerce_int96_tz: Option<String>, default = None
1398+
/// is `None`, but the value is still validated when it is set.
1399+
pub coerce_int96_tz: Option<ConfigTimeZone>, default = None
14001400

14011401
/// (reading) Use any available bloom filters when reading parquet files
14021402
pub bloom_filter_on_read: bool, default = true
@@ -5061,6 +5061,48 @@ mod tests {
50615061
assert_eq!(scalar, DFTimeUnit::Microsecond);
50625062
}
50635063

5064+
#[test]
5065+
fn test_parquet_coerce_int96_tz_validation() {
5066+
use crate::config::ConfigOptions;
5067+
5068+
const KEY: &str = "datafusion.execution.parquet.coerce_int96_tz";
5069+
let mut config = ConfigOptions::default();
5070+
let tz = |config: &ConfigOptions| {
5071+
config
5072+
.execution
5073+
.parquet
5074+
.coerce_int96_tz
5075+
.as_ref()
5076+
.map(|tz| tz.to_string())
5077+
};
5078+
assert_eq!(tz(&config), None);
5079+
5080+
// Valid values are kept exactly as written, with or without coerce_int96.
5081+
for value in ["UTC", "America/Los_Angeles", "+08:00", "-0530", "+08"] {
5082+
config.set(KEY, value).unwrap();
5083+
assert_eq!(tz(&config).as_deref(), Some(value));
5084+
}
5085+
5086+
// A rejected value leaves the previous one in place.
5087+
for value in ["Not/A_Timezone", "utc", "08:00", ""] {
5088+
let err = config.set(KEY, value).unwrap_err();
5089+
assert_contains!(err.to_string(), format!("Invalid time zone: {value}."));
5090+
assert_eq!(tz(&config).as_deref(), Some("+08"));
5091+
}
5092+
5093+
// Reset restores None, and an invalid update leaves it unset.
5094+
config.reset(KEY).unwrap();
5095+
assert_eq!(tz(&config), None);
5096+
assert!(config.set(KEY, "Not/A_Timezone").is_err());
5097+
assert_eq!(tz(&config), None);
5098+
5099+
// The option has no nested fields.
5100+
config.set(KEY, "UTC").unwrap();
5101+
assert!(config.set(&format!("{KEY}.typo"), "UTC").is_err());
5102+
assert!(config.reset(&format!("{KEY}.typo")).is_err());
5103+
assert_eq!(tz(&config).as_deref(), Some("UTC"));
5104+
}
5105+
50645106
#[cfg(feature = "parquet")]
50655107
#[test]
50665108
fn set_cdc_enabled_flag() {

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ use datafusion_session::Session;
6262

6363
use crate::metadata::{DFParquetMetadata, lex_ordering_to_sorting_columns};
6464
use crate::reader::CachedParquetFileReaderFactory;
65-
use crate::source::{ParquetSource, parse_coerce_int96_tz_string};
65+
use crate::source::ParquetSource;
6666
use async_trait::async_trait;
6767
use bytes::Bytes;
6868
use datafusion_datasource::source::DataSourceExec;
@@ -366,8 +366,7 @@ impl FileFormat for ParquetFormat {
366366
.global
367367
.coerce_int96_tz
368368
.as_ref()
369-
.map(|tz| parse_coerce_int96_tz_string(tz))
370-
.transpose()?;
369+
.map(|tz| Arc::<str>::from(tz.as_str()));
371370

372371
let file_metadata_cache =
373372
state.runtime_env().cache_manager.get_file_metadata_cache();
@@ -762,7 +761,7 @@ impl From<&ParquetFormatFactory> for protobuf::TableParquetOptions {
762761
parquet_options::CoerceInt96Opt::CoerceInt96(time_unit.to_string())
763762
}),
764763
coerce_int96_tz_opt: global_options.global.coerce_int96_tz.map(|tz| {
765-
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz)
764+
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz.to_string())
766765
}),
767766
max_predicate_cache_size_opt: global_options.global.max_predicate_cache_size.map(|size| {
768767
parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(size as u64)

‎datafusion/datasource-parquet/src/source.rs‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,7 @@ use datafusion_datasource::as_file_source;
3737
use datafusion_datasource::file_stream::FileOpener;
3838
use datafusion_datasource::morsel::Morselizer;
3939

40-
use arrow::array::timezone::Tz;
4140
use arrow::datatypes::TimeUnit;
42-
use datafusion_common::DataFusionError;
4341
use datafusion_common::config::TableParquetOptions;
4442
use datafusion_common::tree_node::TreeNodeRecursion;
4543
#[cfg(feature = "proto")]
@@ -537,19 +535,6 @@ impl ParquetSource {
537535
}
538536
}
539537

540-
/// Validates that `tz` is a parseable IANA timezone and returns it as an
541-
/// `Arc<str>` for use in `Timestamp(_, Some(tz))` types.
542-
pub(crate) fn parse_coerce_int96_tz_string(
543-
tz: &str,
544-
) -> datafusion_common::Result<Arc<str>> {
545-
tz.parse::<Tz>().map_err(|e| {
546-
DataFusionError::Configuration(format!(
547-
"Invalid parquet coerce_int96_tz {tz:?}: {e}"
548-
))
549-
})?;
550-
Ok(Arc::<str>::from(tz))
551-
}
552-
553538
/// Allows easy conversion from ParquetSource to Arc&lt;dyn FileSource&gt;
554539
impl From<ParquetSource> for Arc<dyn FileSource> {
555540
fn from(source: ParquetSource) -> Self {
@@ -605,8 +590,7 @@ impl FileSource for ParquetSource {
605590
.global
606591
.coerce_int96_tz
607592
.as_ref()
608-
.map(|tz| parse_coerce_int96_tz_string(tz))
609-
.transpose()?;
593+
.map(|tz| Arc::<str>::from(tz.as_str()));
610594
if coerce_int96_tz.is_some() && coerce_int96.is_none() {
611595
warn!(
612596
"coerce_int96_tz is set but coerce_int96 is not; the timezone will be ignored"

‎datafusion/ffi/src/session/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,7 @@ mod tests {
928928
{
929929
table_options.parquet.global.coerce_int96 =
930930
Some(datafusion_common::parquet_config::DFTimeUnit::Microsecond);
931+
table_options.parquet.global.coerce_int96_tz = Some("UTC".parse()?);
931932
}
932933
table_options.current_format = Some(ConfigFileType::JSON);
933934

‎datafusion/proto-common/src/from_proto/mod.rs‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1287,9 +1287,9 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
12871287
coerce_int96: value.coerce_int96_opt.as_ref().map(|opt| match opt {
12881288
protobuf::parquet_options::CoerceInt96Opt::CoerceInt96(v) => v.parse(),
12891289
}).transpose()?,
1290-
coerce_int96_tz: value.coerce_int96_tz_opt.clone().map(|opt| match opt {
1291-
protobuf::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(v) => Some(v),
1292-
}).unwrap_or(None),
1290+
coerce_int96_tz: value.coerce_int96_tz_opt.as_ref().map(|opt| match opt {
1291+
protobuf::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(v) => v.parse(),
1292+
}).transpose()?,
12931293
skip_arrow_metadata: value.skip_arrow_metadata,
12941294
write_row_group_number_distinct_values: value.write_row_group_number_distinct_values,
12951295
max_predicate_cache_size: value.max_predicate_cache_size_opt.map(|opt| match opt {
@@ -1670,16 +1670,27 @@ mod tests {
16701670
assert_eq!(recovered.coerce_int96_tz, None);
16711671
}
16721672

1673+
#[test]
1674+
fn test_parquet_options_coerce_int96_tz_without_unit_round_trip() {
1675+
let opts = ParquetOptions {
1676+
coerce_int96_tz: Some("+08".parse().unwrap()),
1677+
..ParquetOptions::default()
1678+
};
1679+
let recovered = parquet_options_proto_round_trip(opts.clone());
1680+
assert_eq!(recovered.coerce_int96, None);
1681+
assert_eq!(recovered.coerce_int96_tz, opts.coerce_int96_tz);
1682+
}
1683+
16731684
#[test]
16741685
fn test_parquet_options_coerce_int96_tz_set_round_trip() {
16751686
let opts = ParquetOptions {
16761687
coerce_int96: Some(DFTimeUnit::Microsecond),
1677-
coerce_int96_tz: Some("UTC".to_string()),
1688+
coerce_int96_tz: Some("UTC".parse().unwrap()),
16781689
..ParquetOptions::default()
16791690
};
16801691
let recovered = parquet_options_proto_round_trip(opts.clone());
16811692
assert_eq!(recovered.coerce_int96, Some(DFTimeUnit::Microsecond));
1682-
assert_eq!(recovered.coerce_int96_tz, Some("UTC".to_string()));
1693+
assert_eq!(recovered.coerce_int96_tz, opts.coerce_int96_tz);
16831694
}
16841695

16851696
#[test]
@@ -1773,6 +1784,26 @@ mod tests {
17731784
);
17741785
}
17751786

1787+
#[test]
1788+
fn test_invalid_parquet_coerce_int96_tz_rejected_from_proto() {
1789+
// Rejected even when no coerce_int96 unit is set.
1790+
let opts = ParquetOptions::default();
1791+
let mut proto: crate::protobuf_common::ParquetOptions =
1792+
(&opts).try_into().expect("to_proto");
1793+
proto.coerce_int96_tz_opt = Some(
1794+
crate::protobuf_common::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(
1795+
"Not/A_Timezone".to_string(),
1796+
),
1797+
);
1798+
1799+
let err = ParquetOptions::try_from(&proto).unwrap_err();
1800+
assert!(
1801+
err.to_string()
1802+
.contains("Invalid time zone: Not/A_Timezone."),
1803+
"unexpected error: {err}"
1804+
);
1805+
}
1806+
17761807
#[test]
17771808
fn test_invalid_parquet_coerce_int96_rejected_from_proto() {
17781809
let opts = ParquetOptions::default();
@@ -1832,12 +1863,12 @@ mod tests {
18321863
fn test_table_parquet_options_coerce_int96_tz_round_trip() {
18331864
let mut opts = TableParquetOptions::default();
18341865
opts.global.coerce_int96 = Some(DFTimeUnit::Microsecond);
1835-
opts.global.coerce_int96_tz = Some("America/Los_Angeles".to_string());
1866+
opts.global.coerce_int96_tz = Some("America/Los_Angeles".parse().unwrap());
18361867
let recovered = table_parquet_options_proto_round_trip(opts.clone());
18371868
assert_eq!(recovered.global.coerce_int96, Some(DFTimeUnit::Microsecond));
18381869
assert_eq!(
18391870
recovered.global.coerce_int96_tz,
1840-
Some("America/Los_Angeles".to_string())
1871+
opts.global.coerce_int96_tz
18411872
);
18421873
}
18431874

‎datafusion/proto-common/src/to_proto/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -968,7 +968,7 @@ impl TryFrom<&ParquetOptions> for protobuf::ParquetOptions {
968968
skip_arrow_metadata: value.skip_arrow_metadata,
969969
write_row_group_number_distinct_values: value.write_row_group_number_distinct_values,
970970
coerce_int96_opt: value.coerce_int96.map(|v| protobuf::parquet_options::CoerceInt96Opt::CoerceInt96(v.to_string())),
971-
coerce_int96_tz_opt: value.coerce_int96_tz.clone().map(protobuf::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz),
971+
coerce_int96_tz_opt: value.coerce_int96_tz.as_ref().map(|v| protobuf::parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(v.to_string())),
972972
max_predicate_cache_size_opt: value.max_predicate_cache_size.map(|v| protobuf::parquet_options::MaxPredicateCacheSizeOpt::MaxPredicateCacheSize(v as u64)),
973973
max_row_group_bytes_opt: value.max_row_group_bytes.map(|v| protobuf::parquet_options::MaxRowGroupBytesOpt::MaxRowGroupBytes(v.get() as u64)),
974974
content_defined_chunking: Some((&value.content_defined_chunking).into()),

‎datafusion/proto-models/src/from_proto.rs‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -487,10 +487,9 @@ impl TryFrom<&ParquetOptionsProto> for ParquetOptions {
487487
.coerce_int96_tz_opt
488488
.as_ref()
489489
.map(|opt| match opt {
490-
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz) => {
491-
tz.clone()
492-
}
493-
}),
490+
parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(tz) => tz.parse(),
491+
})
492+
.transpose()?,
494493
max_predicate_cache_size: proto
495494
.max_predicate_cache_size_opt
496495
.as_ref()
@@ -618,4 +617,38 @@ mod tests {
618617
.contains("Invalid parquet statistics setting: invalid")
619618
);
620619
}
620+
621+
#[test]
622+
fn accepts_parquet_coerce_int96_tz_without_unit() {
623+
let proto = ParquetOptionsProto {
624+
coerce_int96_tz_opt: Some(parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(
625+
"+08".to_string(),
626+
)),
627+
..Default::default()
628+
};
629+
630+
let options = ParquetOptions::try_from(&proto).unwrap();
631+
assert_eq!(options.coerce_int96, None);
632+
assert_eq!(
633+
options.coerce_int96_tz.as_ref().map(|tz| tz.as_str()),
634+
Some("+08")
635+
);
636+
}
637+
638+
#[test]
639+
fn rejects_invalid_parquet_coerce_int96_tz() {
640+
let proto = ParquetOptionsProto {
641+
coerce_int96_tz_opt: Some(parquet_options::CoerceInt96TzOpt::CoerceInt96Tz(
642+
"Not/A_Timezone".to_string(),
643+
)),
644+
..Default::default()
645+
};
646+
647+
let err = ParquetOptions::try_from(&proto).unwrap_err();
648+
assert!(
649+
err.to_string()
650+
.contains("Invalid time zone: Not/A_Timezone."),
651+
"unexpected error: {err}"
652+
);
653+
}
621654
}

‎datafusion/sqllogictest/test_files/information_schema.slt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ datafusion.execution.parquet.bloom_filter_ndv NULL (writing) Sets bloom filter n
403403
datafusion.execution.parquet.bloom_filter_on_read true (reading) Use any available bloom filters when reading parquet files
404404
datafusion.execution.parquet.bloom_filter_on_write false (writing) Write bloom filters for all columns when creating parquet files
405405
datafusion.execution.parquet.coerce_int96 NULL (reading) If set, parquet reader will read columns of physical type int96 as originating from this resolution instead of nanosecond. Valid values are: ns, us, ms, and s. This is useful for reading data from systems like Spark which store microsecond resolution timestamps in an int96, allowing them to write values with a larger date range than 64-bit timestamps with nanosecond resolution.
406-
datafusion.execution.parquet.coerce_int96_tz NULL (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(<coerce_int96>, Some(<tz>))` instead of the default `Timestamp(<coerce_int96>, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`.
406+
datafusion.execution.parquet.coerce_int96_tz NULL (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(<coerce_int96>, Some(<tz>))` instead of the default `Timestamp(<coerce_int96>, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`, but the value is still validated when it is set.
407407
datafusion.execution.parquet.column_index_truncate_length 64 (writing) Sets column index truncate length
408408
datafusion.execution.parquet.compression zstd(3) (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting.
409409
datafusion.execution.parquet.content_defined_chunking.enabled false (writing) EXPERIMENTAL: Enable content-defined chunking (CDC) when writing parquet files. When enabled, parallel writing is automatically disabled since the chunker state must persist across row groups.

0 commit comments

Comments
 (0)