Skip to content

Commit 14b3edd

Browse files
authored
feat(datetime): support date_bin literal origins for range locality (apache#27135)
## Which issue does this PR close? Closes apache#26913. ## Rationale for this change An explicit origin currently prevents `date_bin` from reusing range partitioning for aggregation, even when its buckets align with every range split. ## What changes are included in this PR? Admit non-null literal origins while retaining timezone-less nanosecond timestamps and positive fixed-duration, month-free strides. A fixed origin shifts the buckets without changing their nondecreasing behavior. The existing boundary proof still rejects crossing buckets and evaluation errors. ## What is the testing strategy for this PR? SQLLogicTests check plans and results for explicit epoch, shifted-aligned, and crossing origins. Unit tests cover rejected arguments, ascending/descending splits, and fail-closed boundary errors. ## Are there any user-facing changes? More aligned `date_bin` aggregations can avoid a hash shuffle. No function-result or public API changes. --- Note: This PR was developed with AI assistance. All changes have been reviewed, and I take full responsibility for this contribution.
1 parent 3b0d0c8 commit 14b3edd

3 files changed

Lines changed: 257 additions & 33 deletions

File tree

‎datafusion/functions/src/datetime/date_bin.rs‎

Lines changed: 75 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ impl ScalarUDFImpl for DateBinFunc {
281281
argument_types: &[DataType],
282282
literal_arguments: &[Option<&ScalarValue>],
283283
) -> bool {
284-
// With the epoch origin, a positive fixed-duration stride rounds down
284+
// With a fixed origin, a positive fixed-duration stride rounds down
285285
// monotonically. As in date_bin_impl, each day is 24 hours.
286286
// Timestamp binning preserves NULLs or returns an error.
287287
match (argument_types, literal_arguments) {
@@ -291,6 +291,18 @@ impl ScalarUDFImpl for DateBinFunc {
291291
Timestamp(Nanosecond, None),
292292
],
293293
[Some(ScalarValue::IntervalMonthDayNano(Some(stride))), None],
294+
)
295+
| (
296+
[
297+
DataType::Interval(MonthDayNano),
298+
Timestamp(Nanosecond, None),
299+
Timestamp(Nanosecond, None),
300+
],
301+
[
302+
Some(ScalarValue::IntervalMonthDayNano(Some(stride))),
303+
None,
304+
Some(ScalarValue::TimestampNanosecond(Some(_), None)),
305+
],
294306
) => {
295307
stride.months == 0
296308
&& TimeDelta::try_days(i64::from(stride.days))
@@ -1013,6 +1025,7 @@ mod tests {
10131025

10141026
let function = DateBinFunc::new();
10151027
let timestamp_type = Timestamp(Nanosecond, None);
1028+
let epoch = ScalarValue::TimestampNanosecond(Some(0), None);
10161029
let cases = [
10171030
(
10181031
"one nanosecond",
@@ -1089,7 +1102,19 @@ mod tests {
10891102
&[Some(&stride), None],
10901103
),
10911104
expected,
1092-
"{description}",
1105+
"{description}, default origin",
1106+
);
1107+
assert_eq!(
1108+
function.supports_range_key_locality_analysis(
1109+
&[
1110+
stride.data_type(),
1111+
timestamp_type.clone(),
1112+
epoch.data_type(),
1113+
],
1114+
&[Some(&stride), None, Some(&epoch)],
1115+
),
1116+
expected,
1117+
"{description}, explicit origin",
10931118
);
10941119
}
10951120
}
@@ -1101,6 +1126,7 @@ mod tests {
11011126

11021127
let function = DateBinFunc::new();
11031128
let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);
1129+
let epoch = ScalarValue::TimestampNanosecond(Some(0), None);
11041130
for source_type in [
11051131
Timestamp(Second, None),
11061132
Timestamp(Millisecond, None),
@@ -1113,7 +1139,14 @@ mod tests {
11131139
&[stride.data_type(), source_type.clone()],
11141140
&[Some(&stride), None],
11151141
),
1116-
"source type {source_type:?}",
1142+
"source type {source_type:?}, default origin",
1143+
);
1144+
assert!(
1145+
!function.supports_range_key_locality_analysis(
1146+
&[stride.data_type(), source_type.clone(), epoch.data_type()],
1147+
&[Some(&stride), None, Some(&epoch)],
1148+
),
1149+
"source type {source_type:?}, explicit origin",
11171150
);
11181151
}
11191152
}
@@ -1131,23 +1164,54 @@ mod tests {
11311164
!function
11321165
.supports_range_key_locality_analysis(&argument_types, &[None, None])
11331166
);
1167+
let epoch = ScalarValue::TimestampNanosecond(Some(0), None);
1168+
assert!(!function.supports_range_key_locality_analysis(
1169+
&[
1170+
argument_types[0].clone(),
1171+
argument_types[1].clone(),
1172+
epoch.data_type(),
1173+
],
1174+
&[None, None, Some(&epoch)],
1175+
));
11341176
}
11351177

11361178
#[test]
1137-
fn range_key_locality_rejects_explicit_origin() {
1179+
fn range_key_locality_rejects_non_literal_origin() {
11381180
let function = DateBinFunc::new();
11391181
let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);
1140-
let epoch = ScalarValue::TimestampNanosecond(Some(0), None);
1182+
let timestamp_type = DataType::Timestamp(TimeUnit::Nanosecond, None);
11411183
assert!(!function.supports_range_key_locality_analysis(
1142-
&[
1143-
stride.data_type(),
1144-
DataType::Timestamp(TimeUnit::Nanosecond, None),
1145-
epoch.data_type(),
1146-
],
1147-
&[Some(&stride), None, Some(&epoch)],
1184+
&[stride.data_type(), timestamp_type.clone(), timestamp_type],
1185+
&[Some(&stride), None, None],
11481186
));
11491187
}
11501188

1189+
#[test]
1190+
fn range_key_locality_rejects_unsupported_origins() {
1191+
let function = DateBinFunc::new();
1192+
let stride = ScalarValue::new_interval_mdn(0, 0, 3_600_000_000_000);
1193+
for origin in [
1194+
ScalarValue::TimestampNanosecond(None, None),
1195+
ScalarValue::TimestampNanosecond(Some(0), Some("UTC".into())),
1196+
ScalarValue::TimestampSecond(Some(0), None),
1197+
ScalarValue::TimestampMillisecond(Some(0), None),
1198+
ScalarValue::TimestampMicrosecond(Some(0), None),
1199+
ScalarValue::Int64(Some(0)),
1200+
] {
1201+
assert!(
1202+
!function.supports_range_key_locality_analysis(
1203+
&[
1204+
stride.data_type(),
1205+
DataType::Timestamp(TimeUnit::Nanosecond, None),
1206+
origin.data_type(),
1207+
],
1208+
&[Some(&stride), None, Some(&origin)],
1209+
),
1210+
"origin {origin:?}",
1211+
);
1212+
}
1213+
}
1214+
11511215
#[test]
11521216
fn output_ordering_requires_constant_stride() {
11531217
use arrow::compute::SortOptions;

‎datafusion/physical-expr/src/partitioning.rs‎

Lines changed: 116 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1383,23 +1383,31 @@ mod tests {
13831383
))
13841384
}
13851385

1386-
/// Builds `date_bin(stride, timestamp)` with a nanosecond-only literal stride
1387-
/// and the default epoch origin. This constructs the expression without evaluating it.
1386+
/// Builds `date_bin` with a nanosecond-only literal stride and optional
1387+
/// timezone-less nanosecond literal origin, without evaluating it.
13881388
fn date_bin_of(
13891389
timestamp: Arc<dyn PhysicalExpr>,
13901390
stride_nanos: i64,
1391+
origin_nanos: Option<i64>,
13911392
) -> Arc<dyn PhysicalExpr> {
1393+
let mut arguments: Vec<Arc<dyn PhysicalExpr>> = vec![
1394+
Arc::new(Literal::new(ScalarValue::new_interval_mdn(
1395+
0,
1396+
0,
1397+
stride_nanos,
1398+
))),
1399+
timestamp,
1400+
];
1401+
if let Some(origin_nanos) = origin_nanos {
1402+
arguments.push(Arc::new(Literal::new(ScalarValue::TimestampNanosecond(
1403+
Some(origin_nanos),
1404+
None,
1405+
))));
1406+
}
13921407
Arc::new(ScalarFunctionExpr::new(
13931408
"date_bin",
13941409
datafusion_functions::datetime::date_bin(),
1395-
vec![
1396-
Arc::new(Literal::new(ScalarValue::new_interval_mdn(
1397-
0,
1398-
0,
1399-
stride_nanos,
1400-
))),
1401-
timestamp,
1402-
],
1410+
arguments,
14031411
Field::new(
14041412
"date_bin",
14051413
DataType::Timestamp(TimeUnit::Nanosecond, None),
@@ -1417,7 +1425,7 @@ mod tests {
14171425
DataType::Timestamp(TimeUnit::Nanosecond, None),
14181426
)])?;
14191427
let hour = 3_600_000_000_000;
1420-
let function = date_bin_of(fixture.col(0), hour);
1428+
let function = date_bin_of(fixture.col(0), hour, None);
14211429
// Ascending compares split - 1 ns with split. Descending compares split
14221430
// with split + 1 ns, so aligned descending splits end a bin rather than start it.
14231431
let cases = [
@@ -1471,7 +1479,7 @@ mod tests {
14711479
let before_split =
14721480
ScalarValue::TimestampNanosecond(Some(first_representable_hour - 1), None);
14731481
let timestamp = fixture.col(0);
1474-
let function = date_bin_of(Arc::clone(&timestamp), hour);
1482+
let function = date_bin_of(Arc::clone(&timestamp), hour, None);
14751483

14761484
let bin_at_split = evaluate_with_range_value(&function, &timestamp, &at_split);
14771485
assert_eq!(bin_at_split, Some(at_split));
@@ -1494,6 +1502,102 @@ mod tests {
14941502
Ok(())
14951503
}
14961504

1505+
#[test]
1506+
fn date_bin_literal_origin_requires_aligned_ascending_splits() -> Result<()> {
1507+
let fixture = PartitioningTestFixture::new(vec![(
1508+
"ts",
1509+
DataType::Timestamp(TimeUnit::Nanosecond, None),
1510+
)])?;
1511+
let hour = 3_600_000_000_000;
1512+
let half_hour = hour / 2;
1513+
let time_bin = date_bin_of(fixture.col(0), hour, Some(half_hour));
1514+
1515+
let aligned_range =
1516+
fixture.range_partitioning([0], vec![timestamp_split(Some(half_hour), None)]);
1517+
assert!(aligned_range.keeps_keys_local(
1518+
&[Arc::clone(&time_bin)],
1519+
&fixture.eq_properties,
1520+
false,
1521+
));
1522+
1523+
let crossing_range =
1524+
fixture.range_partitioning([0], vec![timestamp_split(Some(hour), None)]);
1525+
assert!(!crossing_range.keeps_keys_local(
1526+
&[time_bin],
1527+
&fixture.eq_properties,
1528+
false,
1529+
));
1530+
Ok(())
1531+
}
1532+
1533+
#[test]
1534+
fn date_bin_literal_origin_requires_aligned_descending_splits() -> Result<()> {
1535+
let fixture = PartitioningTestFixture::new(vec![(
1536+
"ts",
1537+
DataType::Timestamp(TimeUnit::Nanosecond, None),
1538+
)])?;
1539+
let hour = 3_600_000_000_000;
1540+
let half_hour = hour / 2;
1541+
let time_bin = date_bin_of(fixture.col(0), hour, Some(half_hour));
1542+
let descending_ordering = LexOrdering::from([fixture.range_sort_expr(
1543+
0,
1544+
SortOptions {
1545+
descending: true,
1546+
nulls_first: true,
1547+
},
1548+
)]);
1549+
1550+
// A descending split belongs to the lower timestamp range. To separate
1551+
// bins at 00:30, put the split at the last nanosecond before 00:30.
1552+
let last_timestamp_before_half_hour = half_hour - 1;
1553+
let aligned_range = fixture.range_partitioning_with_ordering(
1554+
descending_ordering.clone(),
1555+
vec![timestamp_split(Some(last_timestamp_before_half_hour), None)],
1556+
);
1557+
assert!(aligned_range.keeps_keys_local(
1558+
&[Arc::clone(&time_bin)],
1559+
&fixture.eq_properties,
1560+
false,
1561+
));
1562+
1563+
let last_timestamp_before_hour = hour - 1;
1564+
let crossing_range = fixture.range_partitioning_with_ordering(
1565+
descending_ordering,
1566+
vec![timestamp_split(Some(last_timestamp_before_hour), None)],
1567+
);
1568+
assert!(!crossing_range.keeps_keys_local(
1569+
&[time_bin],
1570+
&fixture.eq_properties,
1571+
false,
1572+
));
1573+
Ok(())
1574+
}
1575+
1576+
#[test]
1577+
fn date_bin_literal_origin_rejects_boundary_binning_errors() -> Result<()> {
1578+
let fixture = PartitioningTestFixture::new(vec![(
1579+
"ts",
1580+
DataType::Timestamp(TimeUnit::Nanosecond, None),
1581+
)])?;
1582+
let timestamp = fixture.col(0);
1583+
let function = date_bin_of(Arc::clone(&timestamp), 2, Some(1));
1584+
// A 2 ns stride with origin 1 bins to odd timestamps. i64::MIN is
1585+
// even, so its bin is out of range, but i64::MIN + 1 is representable.
1586+
let split = ScalarValue::TimestampNanosecond(Some(i64::MIN + 1), None);
1587+
let before_split = ScalarValue::TimestampNanosecond(Some(i64::MIN), None);
1588+
assert_eq!(
1589+
evaluate_with_range_value(&function, &timestamp, &split),
1590+
Some(split.clone()),
1591+
);
1592+
assert!(
1593+
evaluate_with_range_value(&function, &timestamp, &before_split).is_none()
1594+
);
1595+
1596+
let range = fixture.range_partitioning([0], vec![SplitPoint::new(vec![split])]);
1597+
assert!(!range.keeps_keys_local(&[function], &fixture.eq_properties, false));
1598+
Ok(())
1599+
}
1600+
14971601
fn timestamp_split(value: Option<i64>, timezone: Option<&str>) -> SplitPoint {
14981602
SplitPoint::new(vec![ScalarValue::TimestampNanosecond(
14991603
value,

0 commit comments

Comments
 (0)