Skip to content

Commit 97728ba

Browse files
committed
fix: handle a NULL format and a literal timestamp with a format column in date_trunc
1 parent 6964f92 commit 97728ba

4 files changed

Lines changed: 152 additions & 6 deletions

File tree

‎native/spark-expr/benches/timestamp_trunc.rs‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use arrow::array::{Array, DictionaryArray, Int32Array, RecordBatch, TimestampMicrosecondArray};
18+
use arrow::array::{
19+
Array, DictionaryArray, Int32Array, RecordBatch, StringArray, TimestampMicrosecondArray,
20+
};
1921
use arrow::datatypes::{DataType, Field, Int32Type, Schema, TimeUnit};
2022
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
2123
use datafusion::physical_expr::expressions::{lit, Column};
@@ -153,6 +155,63 @@ fn criterion_benchmark(c: &mut Criterion) {
153155
}
154156
}
155157
group.finish();
158+
159+
// Formats from a column: one format for every row, and every unit mixed across the rows.
160+
// Daytime instants (16:00 to 22:00 UTC) avoid DST transitions, so older kernels can be
161+
// measured on the same input.
162+
let mut group = c.benchmark_group("timestamp_trunc_format_column");
163+
let rows = 8_192;
164+
let units = [
165+
"YEAR",
166+
"QUARTER",
167+
"MONTH",
168+
"WEEK",
169+
"DAY",
170+
"HOUR",
171+
"MINUTE",
172+
"SECOND",
173+
"MILLISECOND",
174+
"MICROSECOND",
175+
];
176+
for timezone in ["UTC", "America/Los_Angeles"] {
177+
let schema = Arc::new(Schema::new(vec![
178+
Field::new(
179+
"a",
180+
DataType::Timestamp(TimeUnit::Microsecond, Some(timezone.into())),
181+
true,
182+
),
183+
Field::new("fmt", DataType::Utf8, true),
184+
]));
185+
let timestamps = TimestampMicrosecondArray::from_iter_values((0..rows).map(|i| {
186+
BASE_MICROS
187+
+ (i % 366) as i64 * MICROS_PER_DAY
188+
+ 57_600_000_000
189+
+ (i as i64 * 1_234_567) % 21_600_000_000
190+
}))
191+
.with_timezone(timezone);
192+
for (tag, unit) in [("YEAR", Some(0)), ("HOUR", Some(5)), ("mixed", None)] {
193+
let formats = StringArray::from_iter_values(
194+
(0..rows).map(|i| units[unit.unwrap_or(i % units.len())]),
195+
);
196+
let batch = RecordBatch::try_new(
197+
Arc::clone(&schema),
198+
vec![Arc::new(timestamps.clone()), Arc::new(formats)],
199+
)
200+
.unwrap();
201+
let expr = TimestampTruncExpr::new(
202+
Arc::new(Column::new("a", 0)),
203+
Arc::new(Column::new("fmt", 1)),
204+
timezone.to_string(),
205+
true,
206+
);
207+
group.bench_with_input(
208+
BenchmarkId::from_parameter(format!("{timezone}/{tag}/{rows}")),
209+
&batch,
210+
|b, batch| b.iter(|| black_box(expr.evaluate(black_box(batch)).unwrap())),
211+
);
212+
}
213+
}
214+
group.finish();
156215
}
157216

158217
criterion_group!(benches, criterion_benchmark);

‎native/spark-expr/src/datetime_funcs/timestamp_trunc.rs‎

Lines changed: 80 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
// under the License.
1717

1818
use crate::utils::array_with_timezone;
19-
use arrow::array::ArrayRef;
19+
use arrow::array::{new_null_array, ArrayRef};
2020
use arrow::compute::cast;
2121
use arrow::datatypes::{DataType, Schema, TimeUnit::Microsecond};
2222
use arrow::record_batch::RecordBatch;
@@ -142,6 +142,18 @@ impl PhysicalExpr for TimestampTruncExpr {
142142
let scalar = ScalarValue::try_from_array(&relabel(result)?, 0)?;
143143
Ok(ColumnarValue::Scalar(scalar))
144144
}
145+
(ColumnarValue::Scalar(ts_scalar), ColumnarValue::Array(formats)) => {
146+
let ts = ts_scalar.to_array_of_size(formats.len())?;
147+
let result = timestamp_trunc_array_fmt_dyn(&resolve_tz(ts)?, &formats, wrap)?;
148+
Ok(ColumnarValue::Array(relabel(result)?))
149+
}
150+
// A NULL format gives NULL, as in Spark.
151+
(ColumnarValue::Array(ts), ColumnarValue::Scalar(Utf8(None))) => {
152+
Ok(ColumnarValue::Array(new_null_array(&output_type, ts.len())))
153+
}
154+
(ColumnarValue::Scalar(_), ColumnarValue::Scalar(Utf8(None))) => {
155+
Ok(ColumnarValue::Scalar(ScalarValue::try_from(&output_type)?))
156+
}
145157
_ => Err(DataFusionError::Execution(
146158
"Invalid input to function TimestampTrunc. \
147159
Expected (Timestamp, Utf8)"
@@ -184,7 +196,9 @@ fn output_type(input: &DataType) -> DataType {
184196
#[cfg(test)]
185197
mod tests {
186198
use super::*;
187-
use arrow::array::{Array, AsArray, DictionaryArray, Int32Array, TimestampMicrosecondArray};
199+
use arrow::array::{
200+
Array, AsArray, DictionaryArray, Int32Array, StringArray, TimestampMicrosecondArray,
201+
};
188202
use arrow::datatypes::{Field, Int32Type, TimestampMicrosecondType};
189203
use datafusion::physical_expr::expressions::{Column, Literal};
190204

@@ -241,4 +255,68 @@ mod tests {
241255
HOUR_IN_KOLKATA
242256
);
243257
}
258+
259+
/// Evaluates `date_trunc` in America/Los_Angeles with the given timestamp and format inputs.
260+
fn trunc_in_los_angeles(
261+
timestamp: Arc<dyn PhysicalExpr>,
262+
format: Arc<dyn PhysicalExpr>,
263+
batch: &RecordBatch,
264+
) -> ColumnarValue {
265+
TimestampTruncExpr::new(timestamp, format, "America/Los_Angeles".to_string(), false)
266+
.evaluate(batch)
267+
.unwrap()
268+
}
269+
270+
#[test]
271+
fn literal_timestamp_with_a_format_column() {
272+
// 2024-11-03 01:30 PDT, inside the overlap. DAY keeps the input's offset, YEAR does not
273+
// depend on it, and a NULL format gives NULL.
274+
let micros = 1_730_622_600_000_000;
275+
let formats = StringArray::from(vec![Some("DAY"), Some("YEAR"), None]);
276+
let schema = Schema::new(vec![Field::new("fmt", DataType::Utf8, true)]);
277+
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(formats)]).unwrap();
278+
let timestamp = Literal::new(ScalarValue::TimestampMicrosecond(
279+
Some(micros),
280+
Some("UTC".into()),
281+
));
282+
let ColumnarValue::Array(result) =
283+
trunc_in_los_angeles(Arc::new(timestamp), Arc::new(Column::new("fmt", 0)), &batch)
284+
else {
285+
panic!("expected an array");
286+
};
287+
assert_eq!(result.data_type(), &utc_timestamp());
288+
let result = result.as_primitive::<TimestampMicrosecondType>();
289+
// 2024-11-03 00:00 PDT and 2024-01-01 00:00 PST.
290+
assert_eq!(result.value(0), 1_730_617_200_000_000);
291+
assert_eq!(result.value(1), 1_704_096_000_000_000);
292+
assert!(result.is_null(2));
293+
}
294+
295+
#[test]
296+
fn null_literal_format_gives_null() {
297+
let input = TimestampMicrosecondArray::from(vec![Some(MICROS), None]).with_timezone("UTC");
298+
let schema = Schema::new(vec![Field::new("ts", utc_timestamp(), true)]);
299+
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(input)]).unwrap();
300+
let null_format = || Arc::new(Literal::new(Utf8(None)));
301+
let ColumnarValue::Array(result) =
302+
trunc_in_los_angeles(Arc::new(Column::new("ts", 0)), null_format(), &batch)
303+
else {
304+
panic!("expected an array");
305+
};
306+
assert_eq!(result.data_type(), &utc_timestamp());
307+
assert_eq!(result.null_count(), 2);
308+
309+
let timestamp = Arc::new(Literal::new(ScalarValue::TimestampMicrosecond(
310+
Some(MICROS),
311+
Some("UTC".into()),
312+
)));
313+
let ColumnarValue::Scalar(result) = trunc_in_los_angeles(timestamp, null_format(), &batch)
314+
else {
315+
panic!("expected a scalar");
316+
};
317+
assert_eq!(
318+
result,
319+
ScalarValue::TimestampMicrosecond(None, Some("UTC".into()))
320+
);
321+
}
244322
}

‎native/spark-expr/src/kernels/temporal.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -896,9 +896,10 @@ where
896896
Ok(result)
897897
}
898898

899-
/// The zone-aware scalar-format fallback for HOUR/DAY outside DataFusion 55.1's internal
900-
/// TimestampNanosecond range. UTC/NTZ coarse units use integer calendar arithmetic instead.
901-
/// Row-format paths continue to call the same underlying helpers.
899+
/// The zone-aware fallback for HOUR/DAY outside DataFusion 55.1's internal TimestampNanosecond
900+
/// range, for literal formats and format columns alike. UTC/NTZ coarse units use integer calendar
901+
/// arithmetic instead. Like every native local-time conversion, it stops applying DST after
902+
/// chrono-tz's last transition, around 2099, while Spark keeps applying the zone's rules (#6816).
902903
fn timestamp_trunc_legacy(
903904
array: &TimestampMicrosecondArray,
904905
format: &str,

‎spark/src/test/resources/sql-tests/expressions/datetime/trunc_timestamp_dst_format_column.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,11 @@ INSERT INTO test_trunc_fmt SELECT ts, fmt FROM test_trunc_fmt_ts CROSS JOIN test
6464

6565
query expect_native(date_trunc)
6666
SELECT ts, fmt, date_trunc(fmt, ts) FROM test_trunc_fmt ORDER BY ts, fmt
67+
68+
-- A NULL literal format gives NULL.
69+
query expect_native(date_trunc)
70+
SELECT ts, date_trunc(NULL, ts) FROM test_trunc_fmt_ts ORDER BY ts
71+
72+
-- A literal timestamp with the format in a column. 2024-11-03 01:30 is in the US fall-back overlap.
73+
query expect_native(date_trunc)
74+
SELECT fmt, date_trunc(fmt, TIMESTAMP '2024-11-03 01:30:00') FROM test_trunc_fmt_unit ORDER BY fmt

0 commit comments

Comments
 (0)