Skip to content

Commit ce04bf9

Browse files
robert3005claude
andauthored
fix(datetime-parts): keep null rows null and order by parts (#10145)
Combining the nullable days comparison with the non-nullable seconds and subseconds comparisons under Kleene logic turned a null row into `false` or `true` whenever its stored parts differed from the constant. The kernel now restores the array's validity on the combined result. Ordering comparisons previously engaged only when every day was strictly on one side of the constant and otherwise decoded the array. The parts of a timestamp order lexicographically, so the kernel now combines the day, second, and subsecond comparisons for every ordering operator, keeping the early exits. Signed-off-by: Claude <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent ed119bf commit ce04bf9

1 file changed

Lines changed: 220 additions & 52 deletions

File tree

‎encodings/datetime-parts/src/compute/compare.rs‎

Lines changed: 220 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -51,32 +51,45 @@ impl CompareKernel for DateTimeParts {
5151
};
5252
let ts_parts = timestamp::split(timestamp, options.unit)?;
5353

54-
match operator {
55-
CompareOperator::Eq => compare_eq(lhs, &ts_parts, nullability, ctx),
56-
CompareOperator::NotEq => compare_ne(lhs, &ts_parts, nullability, ctx),
57-
// lt and lte have identical behavior, as we optimize
58-
// for the case that all days on the lhs are smaller.
59-
// If that special case is not hit, we return `Ok(None)` to
60-
// signal that the comparison wasn't handled within dtp.
61-
CompareOperator::Lt => compare_lt(lhs, &ts_parts, nullability, ctx),
62-
CompareOperator::Lte => compare_lt(lhs, &ts_parts, nullability, ctx),
63-
// (Like for lt, lte)
64-
CompareOperator::Gt => compare_gt(lhs, &ts_parts, nullability, ctx),
65-
CompareOperator::Gte => compare_gt(lhs, &ts_parts, nullability, ctx),
66-
}
54+
let result = match operator {
55+
CompareOperator::Eq => compare_eq(lhs, &ts_parts, nullability, ctx)?,
56+
CompareOperator::NotEq => compare_ne(lhs, &ts_parts, nullability, ctx)?,
57+
CompareOperator::Lt
58+
| CompareOperator::Lte
59+
| CompareOperator::Gt
60+
| CompareOperator::Gte => compare_ordered(lhs, &ts_parts, operator, nullability, ctx)?,
61+
};
62+
with_validity(lhs, result, nullability).map(Some)
6763
}
6864
}
6965

66+
/// Restores the array's validity on a result combined from its parts.
67+
///
68+
/// The seconds and subseconds are non-nullable, so combining their comparisons with the nullable
69+
/// days comparison under Kleene logic can turn a null row into `false` or `true`.
70+
fn with_validity(
71+
lhs: ArrayView<DateTimeParts>,
72+
result: ArrayRef,
73+
nullability: Nullability,
74+
) -> VortexResult<ArrayRef> {
75+
Ok(match lhs.validity()?.union_nullability(nullability) {
76+
Validity::NonNullable | Validity::AllValid => result,
77+
Validity::AllInvalid => {
78+
ConstantArray::new(Scalar::null(DType::Bool(nullability)), lhs.len()).into_array()
79+
}
80+
Validity::Array(validity) => result.mask(validity)?,
81+
})
82+
}
83+
7084
fn compare_eq(
7185
lhs: ArrayView<DateTimeParts>,
7286
ts_parts: &timestamp::TimestampParts,
7387
nullability: Nullability,
7488
ctx: &mut ExecutionCtx,
75-
) -> VortexResult<Option<ArrayRef>> {
89+
) -> VortexResult<ArrayRef> {
7690
let mut comparison = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Eq, nullability)?;
7791
if comparison.statistics().compute_max::<bool>(ctx) == Some(false) {
78-
// All values are different.
79-
return Ok(Some(comparison));
92+
return Ok(comparison);
8093
}
8194

8295
comparison = compare_dtp(
@@ -88,36 +101,32 @@ fn compare_eq(
88101
.binary(comparison, Operator::And)?;
89102

90103
if comparison.statistics().compute_max::<bool>(ctx) == Some(false) {
91-
// All values are different.
92-
return Ok(Some(comparison));
104+
return Ok(comparison);
93105
}
94106

95-
comparison = compare_dtp(
107+
compare_dtp(
96108
lhs.subseconds(),
97109
ts_parts.subseconds,
98110
CompareOperator::Eq,
99111
nullability,
100112
)?
101-
.binary(comparison, Operator::And)?;
102-
103-
Ok(Some(comparison))
113+
.binary(comparison, Operator::And)
104114
}
105115

106116
fn compare_ne(
107117
lhs: ArrayView<DateTimeParts>,
108118
ts_parts: &timestamp::TimestampParts,
109119
nullability: Nullability,
110120
ctx: &mut ExecutionCtx,
111-
) -> VortexResult<Option<ArrayRef>> {
121+
) -> VortexResult<ArrayRef> {
112122
let mut comparison = compare_dtp(
113123
lhs.days(),
114124
ts_parts.days,
115125
CompareOperator::NotEq,
116126
nullability,
117127
)?;
118128
if comparison.statistics().compute_min::<bool>(ctx) == Some(true) {
119-
// All values are different.
120-
return Ok(Some(comparison));
129+
return Ok(comparison);
121130
}
122131

123132
comparison = compare_dtp(
@@ -129,49 +138,59 @@ fn compare_ne(
129138
.binary(comparison, Operator::Or)?;
130139

131140
if comparison.statistics().compute_min::<bool>(ctx) == Some(true) {
132-
// All values are different.
133-
return Ok(Some(comparison));
141+
return Ok(comparison);
134142
}
135143

136-
comparison = compare_dtp(
144+
compare_dtp(
137145
lhs.subseconds(),
138146
ts_parts.subseconds,
139147
CompareOperator::NotEq,
140148
nullability,
141149
)?
142-
.binary(comparison, Operator::Or)?;
143-
144-
Ok(Some(comparison))
150+
.binary(comparison, Operator::Or)
145151
}
146152

147-
fn compare_lt(
153+
/// Orders timestamps by their parts.
154+
///
155+
/// `split` gives every timestamp a unique `(days, seconds, subseconds)` triple whose remainders
156+
/// share the timestamp's sign, so the parts compare lexicographically: the first part that
157+
/// differs decides, and the last part carries the operator's strictness.
158+
fn compare_ordered(
148159
lhs: ArrayView<DateTimeParts>,
149160
ts_parts: &timestamp::TimestampParts,
161+
operator: CompareOperator,
150162
nullability: Nullability,
151163
ctx: &mut ExecutionCtx,
152-
) -> VortexResult<Option<ArrayRef>> {
153-
let days_lt = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Lt, nullability)?;
154-
if days_lt.statistics().compute_min::<bool>(ctx) == Some(true) {
155-
// All values on the lhs are smaller.
156-
return Ok(Some(days_lt));
157-
}
158-
159-
Ok(None)
160-
}
164+
) -> VortexResult<ArrayRef> {
165+
let strict = match operator {
166+
CompareOperator::Lt | CompareOperator::Lte => CompareOperator::Lt,
167+
CompareOperator::Gt | CompareOperator::Gte => CompareOperator::Gt,
168+
CompareOperator::Eq | CompareOperator::NotEq => {
169+
unreachable!("equality is answered by compare_eq and compare_ne")
170+
}
171+
};
161172

162-
fn compare_gt(
163-
lhs: ArrayView<DateTimeParts>,
164-
ts_parts: &timestamp::TimestampParts,
165-
nullability: Nullability,
166-
ctx: &mut ExecutionCtx,
167-
) -> VortexResult<Option<ArrayRef>> {
168-
let days_gt = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Gt, nullability)?;
169-
if days_gt.statistics().compute_min::<bool>(ctx) == Some(true) {
170-
// All values on the lhs are larger.
171-
return Ok(Some(days_gt));
173+
let days_strict = compare_dtp(lhs.days(), ts_parts.days, strict, nullability)?;
174+
if days_strict.statistics().compute_min::<bool>(ctx) == Some(true) {
175+
return Ok(days_strict);
172176
}
177+
let days_eq = compare_dtp(lhs.days(), ts_parts.days, CompareOperator::Eq, nullability)?;
178+
if days_eq.statistics().compute_max::<bool>(ctx) == Some(false) {
179+
return Ok(days_strict);
180+
}
181+
182+
let seconds_strict = compare_dtp(lhs.seconds(), ts_parts.seconds, strict, nullability)?;
183+
let seconds_eq = compare_dtp(
184+
lhs.seconds(),
185+
ts_parts.seconds,
186+
CompareOperator::Eq,
187+
nullability,
188+
)?;
189+
let subseconds = compare_dtp(lhs.subseconds(), ts_parts.subseconds, operator, nullability)?;
173190

174-
Ok(None)
191+
let within_day =
192+
seconds_strict.binary(seconds_eq.binary(subseconds, Operator::And)?, Operator::Or)?;
193+
days_strict.binary(days_eq.binary(within_day, Operator::And)?, Operator::Or)
175194
}
176195

177196
fn compare_dtp(
@@ -243,6 +262,7 @@ mod test {
243262
use vortex_array::extension::datetime::Timestamp;
244263
use vortex_array::extension::datetime::TimestampOptions;
245264
use vortex_array::scalar::Scalar;
265+
use vortex_array::scalar_fn::fns::binary::CompareKernel;
246266
use vortex_array::validity::Validity;
247267
use vortex_buffer::buffer;
248268

@@ -510,4 +530,152 @@ mod test {
510530
assert_arrays_eq!(gt, BoolArray::from_iter(expected), &mut ctx);
511531
Ok(())
512532
}
533+
534+
const ALL_OPERATORS: [CompareOperator; 6] = [
535+
CompareOperator::Eq,
536+
CompareOperator::NotEq,
537+
CompareOperator::Lt,
538+
CompareOperator::Lte,
539+
CompareOperator::Gt,
540+
CompareOperator::Gte,
541+
];
542+
543+
fn timestamp_constant(array: &DateTimePartsArray, timestamp: i64, len: usize) -> ArrayRef {
544+
let options = array
545+
.dtype()
546+
.as_extension()
547+
.metadata_opt::<Timestamp>()
548+
.expect("timestamp metadata")
549+
.clone();
550+
ConstantArray::new(
551+
Scalar::extension::<Timestamp>(
552+
options,
553+
Scalar::primitive(timestamp, Nullability::NonNullable),
554+
),
555+
len,
556+
)
557+
.into_array()
558+
}
559+
560+
/// Asserts the kernel engages for every operator and matches the comparison over the
561+
/// timestamps it was built from.
562+
fn assert_matches_temporal(
563+
timestamps: PrimitiveArray,
564+
unit: TimeUnit,
565+
constants: impl IntoIterator<Item = i64>,
566+
) -> VortexResult<()> {
567+
let mut ctx = array_session().create_execution_ctx();
568+
let temporal =
569+
TemporalArray::new_timestamp(timestamps.into_array(), unit, Some("UTC".into()));
570+
let array = DateTimeParts::try_from_temporal(temporal.clone(), &mut ctx)?;
571+
for constant in constants {
572+
let rhs = timestamp_constant(&array, constant, array.len());
573+
for operator in ALL_OPERATORS {
574+
let actual = <DateTimeParts as CompareKernel>::compare(
575+
array.as_view(),
576+
&rhs,
577+
operator,
578+
&mut ctx,
579+
)?
580+
.unwrap_or_else(|| panic!("datetime-parts compare must engage for {operator:?}"));
581+
let expected = temporal
582+
.clone()
583+
.into_array()
584+
.binary(rhs.clone(), Operator::from(operator))?;
585+
assert_arrays_eq!(actual, expected, &mut ctx);
586+
}
587+
}
588+
Ok(())
589+
}
590+
591+
#[rstest]
592+
#[case::seconds(TimeUnit::Seconds)]
593+
#[case::milliseconds(TimeUnit::Milliseconds)]
594+
#[case::nanoseconds(TimeUnit::Nanoseconds)]
595+
fn ordering_matches_temporal(#[case] unit: TimeUnit) -> VortexResult<()> {
596+
let divisor: i64 = match unit {
597+
TimeUnit::Seconds => 1,
598+
TimeUnit::Milliseconds => 1_000,
599+
TimeUnit::Microseconds => 1_000_000,
600+
TimeUnit::Nanoseconds => 1_000_000_000,
601+
TimeUnit::Days => unreachable!(),
602+
};
603+
let day = 86_400 * divisor;
604+
// Both signs, rows sharing a day or a second, and rows differing only in subseconds.
605+
let values: Vec<i64> = vec![
606+
-2 * day - 1,
607+
-day - divisor,
608+
-day,
609+
-day + 1,
610+
-divisor - 1,
611+
-1,
612+
0,
613+
1,
614+
divisor - 1,
615+
divisor,
616+
day - 1,
617+
day,
618+
day + divisor,
619+
day + divisor + 1,
620+
3 * day + 17 * divisor + 5,
621+
];
622+
let mut constants = Vec::new();
623+
for value in &values {
624+
constants.extend([value - 1, *value, value + 1]);
625+
}
626+
constants.extend([-20_000 * day, 20_000 * day]);
627+
628+
assert_matches_temporal(
629+
PrimitiveArray::from_iter(values.clone()),
630+
unit,
631+
constants.clone(),
632+
)?;
633+
634+
let nullable = PrimitiveArray::from_option_iter(
635+
values
636+
.iter()
637+
.enumerate()
638+
.map(|(i, value)| (i % 3 != 1).then_some(*value)),
639+
);
640+
assert_matches_temporal(nullable, unit, constants)
641+
}
642+
643+
#[test]
644+
fn null_rows_stay_null_when_parts_differ() -> VortexResult<()> {
645+
let mut ctx = array_session().create_execution_ctx();
646+
// Row 1 is null and its stored seconds differ from the constant's.
647+
let values = PrimitiveArray::new(
648+
buffer![86_400i64 + 5, 86_400i64 + 7, 86_400i64 + 5],
649+
Validity::from_iter([true, false, true]),
650+
);
651+
let array = DateTimeParts::try_from_temporal(
652+
TemporalArray::new_timestamp(
653+
values.into_array(),
654+
TimeUnit::Seconds,
655+
Some("UTC".into()),
656+
),
657+
&mut ctx,
658+
)?;
659+
let rhs = timestamp_constant(&array, 86_400 + 5, array.len());
660+
661+
for (operator, valid) in [
662+
(CompareOperator::Eq, true),
663+
(CompareOperator::NotEq, false),
664+
(CompareOperator::Lt, false),
665+
(CompareOperator::Lte, true),
666+
(CompareOperator::Gt, false),
667+
(CompareOperator::Gte, true),
668+
] {
669+
let actual = <DateTimeParts as CompareKernel>::compare(
670+
array.as_view(),
671+
&rhs,
672+
operator,
673+
&mut ctx,
674+
)?
675+
.expect("kernel engages");
676+
let expected = BoolArray::from_iter([Some(valid), None, Some(valid)]);
677+
assert_arrays_eq!(actual, expected, &mut ctx);
678+
}
679+
Ok(())
680+
}
513681
}

0 commit comments

Comments
 (0)