Skip to content

Commit 13dd764

Browse files
committed
fix: avoid invalid date_bin ordering after overflow
1 parent 2306a4b commit 13dd764

2 files changed

Lines changed: 37 additions & 8 deletions

File tree

‎datafusion/functions/src/datetime/date_bin.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -269,14 +269,22 @@ impl ScalarUDFImpl for DateBinFunc {
269269
}
270270

271271
fn output_ordering(&self, input: &[ExprProperties]) -> Result<SortProperties> {
272-
// The DATE_BIN function preserves the order of its second argument.
273272
let step = &input[0];
274273
let date_value = &input[1];
275274
let reference = input.get(2);
276275

277-
if step.sort_properties.eq(&SortProperties::Singleton)
276+
// Scaling these representations to nanoseconds can overflow and turn
277+
// otherwise valid input rows into NULL. The generated NULLs need not
278+
// have the same placement as the source ordering.
279+
let scale_can_overflow = matches!(
280+
date_value.range.data_type(),
281+
Timestamp(Second | Millisecond | Microsecond, _) | Time64(Microsecond)
282+
);
283+
284+
if !scale_can_overflow
285+
&& step.sort_properties == SortProperties::Singleton
278286
&& reference
279-
.map(|r| r.sort_properties.eq(&SortProperties::Singleton))
287+
.map(|r| r.sort_properties == SortProperties::Singleton)
280288
.unwrap_or(true)
281289
{
282290
Ok(date_value.sort_properties)

‎datafusion/sqllogictest/test_files/datetime/timestamps.slt‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1728,7 +1728,7 @@ physical_plan
17281728
02)--SortExec: TopK(fetch=2), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false]
17291729
03)----DataSourceExec: partitions=1, partition_sizes=[1]
17301730

1731-
# Typed bounds allow date_trunc to propagate ordering through timezone-free date_bin
1731+
# date_bin remains conservatively unordered because per-row failures become NULL
17321732
query TT
17331733
EXPLAIN SELECT date_trunc('hour', date_bin(INTERVAL '1 minute', ts)) AS truncated
17341734
FROM (
@@ -1750,10 +1750,31 @@ logical_plan
17501750
04)------Projection: CAST(column1 AS Timestamp(s)) AS ts
17511751
05)--------Values: (Int64(7200)), (Int64(3600))
17521752
physical_plan
1753-
01)ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
1754-
02)--SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
1755-
03)----ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
1756-
04)------DataSourceExec: partitions=1, partition_sizes=[1]
1753+
01)SortExec: expr=[truncated@0 ASC NULLS LAST], preserve_partitioning=[false]
1754+
02)--ProjectionExec: expr=[date_trunc(hour, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 60000000000 }, ts@0)) as truncated]
1755+
03)----SortExec: TopK(fetch=2), expr=[ts@0 ASC NULLS LAST], preserve_partitioning=[false]
1756+
04)------ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts]
1757+
05)--------DataSourceExec: partitions=1, partition_sizes=[1]
1758+
1759+
# date_bin can return NULL for out-of-range values, so its output does not
1760+
# necessarily preserve the source's NULL ordering.
1761+
query P
1762+
SELECT date_trunc('second', date_bin(INTERVAL '1 second', ts)) AS b
1763+
FROM (
1764+
SELECT arrow_cast(column1, 'Timestamp(Second)') AS ts
1765+
FROM (VALUES
1766+
(-10000000000::bigint),
1767+
(0::bigint),
1768+
(10000000000::bigint)
1769+
)
1770+
ORDER BY ts
1771+
LIMIT 3
1772+
)
1773+
ORDER BY b ASC NULLS LAST
1774+
----
1775+
1970-01-01T00:00:00
1776+
NULL
1777+
NULL
17571778

17581779
statement ok
17591780
CREATE TABLE timestamp_strings(ts varchar)

0 commit comments

Comments
 (0)