Skip to content

Commit 916029d

Browse files
committed
refactor: introduce Int96Coercer builder; revert tz lowercasing
- Replace coerce_int96_to_resolution_with_tz with an Int96Coercer builder per @alamb's suggestion. The existing coerce_int96_to_resolution stays as a deprecated thin wrapper for backwards compatibility. - Revert the str::to_lowercase transform on coerce_int96_tz: IANA timezones are case-sensitive (verified that Tz::from_str rejects "utc" and "america/los_angeles"), so lowercasing would break valid values. - Add a comment on the coerce_int96_tz: None default in the opener.rs test builder.
1 parent 3a8e248 commit 916029d

4 files changed

Lines changed: 111 additions & 39 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -916,7 +916,7 @@ config_namespace! {
916916
/// Arrow type to be timezone-aware (e.g. for Spark `TimestampType`
917917
/// semantics) should set this to `"UTC"`. No effect when `coerce_int96`
918918
/// is `None`.
919-
pub coerce_int96_tz: Option<String>, transform = str::to_lowercase, default = None
919+
pub coerce_int96_tz: Option<String>, default = None
920920

921921
/// (reading) Use any available bloom filters when reading parquet files
922922
pub bloom_filter_on_read: bool, default = true

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 97 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -712,28 +712,102 @@ pub fn apply_file_schema_type_coercions(
712712
))
713713
}
714714

715-
/// Coerces the file schema's Timestamps to the provided TimeUnit if Parquet schema contains INT96.
715+
/// Coerces the file schema's Timestamps to the provided TimeUnit if the
716+
/// Parquet schema contains INT96.
716717
///
717-
/// Equivalent to calling [`coerce_int96_to_resolution_with_tz`] with `timezone: None`,
718-
/// producing `Timestamp(time_unit, None)` for INT96-derived columns (the historical
719-
/// default). Use [`coerce_int96_to_resolution_with_tz`] to attach a timezone.
718+
/// Deprecated wrapper around [`Int96Coercer`]; use the builder directly
719+
/// instead — it also supports attaching a timezone via
720+
/// [`Int96Coercer::with_timezone`].
721+
#[deprecated(since = "53.2.0", note = "use `Int96Coercer` instead")]
720722
pub fn coerce_int96_to_resolution(
721723
parquet_schema: &SchemaDescriptor,
722724
file_schema: &Schema,
723725
time_unit: &TimeUnit,
724726
) -> Option<Schema> {
725-
coerce_int96_to_resolution_with_tz(parquet_schema, file_schema, time_unit, None)
727+
Int96Coercer::new(parquet_schema, file_schema, time_unit).coerce()
726728
}
727729

728-
/// Coerces the file schema's Timestamps to the provided TimeUnit if Parquet schema contains INT96.
730+
/// Builder for coercing INT96-originated Timestamp columns in `file_schema`
731+
/// to a specific [`TimeUnit`], optionally attaching a timezone.
729732
///
730-
/// When `timezone` is `Some`, INT96-derived columns coerce to
731-
/// `Timestamp(time_unit, Some(timezone))`; otherwise they coerce to
732-
/// `Timestamp(time_unit, None)` (the historical default). Spark and other
733-
/// systems write INT96 as UTC-adjusted instants, so callers that need the
734-
/// resulting Arrow type to be timezone-aware should pass
735-
/// `Some(&Arc::from("UTC"))`.
736-
pub fn coerce_int96_to_resolution_with_tz(
733+
/// INT96 is the legacy Parquet representation that systems like Spark use for
734+
/// timestamps. Arrow surfaces it as `Timestamp(Nanosecond, None)`, but the
735+
/// underlying values are written as UTC-adjusted instants. Use this builder
736+
/// to:
737+
///
738+
/// - Coerce INT96-derived columns to a smaller [`TimeUnit`] (e.g. microseconds)
739+
/// to extend the representable date range.
740+
/// - Optionally attach a timezone so the resulting Arrow type carries the
741+
/// timezone-aware semantic (`Timestamp(unit, Some(tz))`). Without a
742+
/// timezone, INT96-derived columns become `Timestamp(unit, None)` — the
743+
/// historical default.
744+
///
745+
/// Returns `None` if `file_schema` contains no INT96-derived columns.
746+
///
747+
/// # Example
748+
///
749+
/// ```ignore
750+
/// use std::sync::Arc;
751+
/// use arrow::datatypes::TimeUnit;
752+
/// use datafusion_datasource_parquet::Int96Coercer;
753+
///
754+
/// let coerced = Int96Coercer::new(parquet_schema, file_schema, &TimeUnit::Microsecond)
755+
/// .with_timezone(Some(Arc::from("UTC")))
756+
/// .coerce();
757+
/// ```
758+
pub struct Int96Coercer<'a> {
759+
parquet_schema: &'a SchemaDescriptor,
760+
file_schema: &'a Schema,
761+
time_unit: &'a TimeUnit,
762+
timezone: Option<Arc<str>>,
763+
}
764+
765+
impl<'a> Int96Coercer<'a> {
766+
/// Create a new builder. INT96-derived columns will coerce to
767+
/// `Timestamp(time_unit, None)` unless [`Self::with_timezone`] is set.
768+
pub fn new(
769+
parquet_schema: &'a SchemaDescriptor,
770+
file_schema: &'a Schema,
771+
time_unit: &'a TimeUnit,
772+
) -> Self {
773+
Self {
774+
parquet_schema,
775+
file_schema,
776+
time_unit,
777+
timezone: None,
778+
}
779+
}
780+
781+
/// Attach a timezone to INT96-derived columns. When `Some`, INT96-derived
782+
/// columns coerce to `Timestamp(time_unit, Some(timezone))` instead of
783+
/// the default `Timestamp(time_unit, None)`. Spark and other systems
784+
/// write INT96 as UTC-adjusted instants, so callers that need the
785+
/// resulting Arrow type to be timezone-aware should pass
786+
/// `Some(Arc::from("UTC"))`.
787+
pub fn with_timezone(mut self, timezone: Option<Arc<str>>) -> Self {
788+
self.timezone = timezone;
789+
self
790+
}
791+
792+
/// Run the coercion, returning the rewritten schema or `None` if
793+
/// `file_schema` contains no INT96-derived columns.
794+
pub fn coerce(self) -> Option<Schema> {
795+
let Self {
796+
parquet_schema,
797+
file_schema,
798+
time_unit,
799+
timezone,
800+
} = self;
801+
coerce_int96_to_resolution_impl(
802+
parquet_schema,
803+
file_schema,
804+
time_unit,
805+
timezone.as_ref(),
806+
)
807+
}
808+
}
809+
810+
fn coerce_int96_to_resolution_impl(
737811
parquet_schema: &SchemaDescriptor,
738812
file_schema: &Schema,
739813
time_unit: &TimeUnit,
@@ -1872,9 +1946,9 @@ mod tests {
18721946

18731947
let arrow_schema = parquet_to_arrow_schema(&descr, None).unwrap();
18741948

1875-
let result =
1876-
coerce_int96_to_resolution(&descr, &arrow_schema, &TimeUnit::Microsecond)
1877-
.unwrap();
1949+
let result = Int96Coercer::new(&descr, &arrow_schema, &TimeUnit::Microsecond)
1950+
.coerce()
1951+
.unwrap();
18781952

18791953
// Only the first field (c0) should be converted to a microsecond timestamp because it's the
18801954
// only timestamp that originated from an INT96.
@@ -1925,14 +1999,10 @@ mod tests {
19251999

19262000
let arrow_schema = parquet_to_arrow_schema(&descr, None).unwrap();
19272001

1928-
let tz: Arc<str> = Arc::from("UTC");
1929-
let result = coerce_int96_to_resolution_with_tz(
1930-
&descr,
1931-
&arrow_schema,
1932-
&TimeUnit::Microsecond,
1933-
Some(&tz),
1934-
)
1935-
.unwrap();
2002+
let result = Int96Coercer::new(&descr, &arrow_schema, &TimeUnit::Microsecond)
2003+
.with_timezone(Some(Arc::from("UTC")))
2004+
.coerce()
2005+
.unwrap();
19362006

19372007
let expected_schema = Schema::new(vec![
19382008
Field::new(
@@ -2019,9 +2089,9 @@ mod tests {
20192089

20202090
let arrow_schema = parquet_to_arrow_schema(&descr, None).unwrap();
20212091

2022-
let result =
2023-
coerce_int96_to_resolution(&descr, &arrow_schema, &TimeUnit::Microsecond)
2024-
.unwrap();
2092+
let result = Int96Coercer::new(&descr, &arrow_schema, &TimeUnit::Microsecond)
2093+
.coerce()
2094+
.unwrap();
20252095

20262096
let expected_schema = Schema::new(vec![
20272097
Field::new("c0", DataType::Timestamp(TimeUnit::Microsecond, None), true),

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
//! [`DFParquetMetadata`] for fetching Parquet file metadata, statistics
1919
//! and schema information.
2020
21-
use crate::{apply_file_schema_type_coercions, coerce_int96_to_resolution_with_tz};
21+
use crate::{Int96Coercer, apply_file_schema_type_coercions};
2222
use arrow::array::{Array, ArrayRef, BooleanArray};
2323
use arrow::compute::and;
2424
use arrow::compute::kernels::cmp::eq;
@@ -227,12 +227,9 @@ impl<'a> DFParquetMetadata<'a> {
227227
.coerce_int96
228228
.as_ref()
229229
.and_then(|time_unit| {
230-
coerce_int96_to_resolution_with_tz(
231-
file_metadata.schema_descr(),
232-
&schema,
233-
time_unit,
234-
self.coerce_int96_tz.as_ref(),
235-
)
230+
Int96Coercer::new(file_metadata.schema_descr(), &schema, time_unit)
231+
.with_timezone(self.coerce_int96_tz.clone())
232+
.coerce()
236233
})
237234
.unwrap_or(schema);
238235
Ok(schema)

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ use crate::push_decoder::{DecoderBuilderConfig, PushDecoderStreamState};
2323
use crate::row_filter::{RowFilterGenerator, build_projection_read_plan};
2424
use crate::row_group_filter::{BloomFilterStatistics, RowGroupAccessPlanFilter};
2525
use crate::{
26-
ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,
27-
apply_file_schema_type_coercions, coerce_int96_to_resolution_with_tz,
26+
Int96Coercer, ParquetAccessPlan, ParquetFileMetrics, ParquetFileReaderFactory,
27+
apply_file_schema_type_coercions,
2828
};
2929
use arrow::array::RecordBatch;
3030
use arrow::datatypes::DataType;
@@ -784,12 +784,13 @@ impl MetadataLoadedParquetOpen {
784784
}
785785

786786
if let Some(ref coerce) = prepared.coerce_int96
787-
&& let Some(merged) = coerce_int96_to_resolution_with_tz(
787+
&& let Some(merged) = Int96Coercer::new(
788788
reader_metadata.parquet_schema(),
789789
&physical_file_schema,
790790
coerce,
791-
prepared.coerce_int96_tz.as_ref(),
792791
)
792+
.with_timezone(prepared.coerce_int96_tz.clone())
793+
.coerce()
793794
{
794795
physical_file_schema = Arc::new(merged);
795796
options = options.with_schema(Arc::clone(&physical_file_schema));
@@ -1755,6 +1756,10 @@ mod test {
17551756
enable_bloom_filter: self.enable_bloom_filter,
17561757
enable_row_group_stats_pruning: self.enable_row_group_stats_pruning,
17571758
coerce_int96: self.coerce_int96,
1759+
// No tests currently exercise coerce_int96_tz; the existing
1760+
// coerce_int96 tests all expect the legacy `Timestamp(_, None)`
1761+
// output. If a future test needs to set a timezone, add a
1762+
// builder setter analogous to with_coerce_int96.
17581763
coerce_int96_tz: None,
17591764
#[cfg(feature = "parquet_encryption")]
17601765
file_decryption_properties: None,

0 commit comments

Comments
 (0)