Skip to content

Commit e8974d6

Browse files
authored
fix(datetime-parts): keep constant nullability in comparison pushdown (#10512)
Fixes #10481. `DTPComparisonPushDownRule` replaces a midnight timestamp constant with a days constant. It cast that constant to the dtype of `days`. When the column is non-nullable and the constant is nullable, the rewritten comparison returned `bool`, but the original comparison returned `bool?`. The reduce then failed with `Reduced array dtype mismatch`. vortex-datafusion makes all literals nullable. Thus a midnight literal (for example `ts >= TIMESTAMP '2026-10-05T00:00:00Z'`) against a non-nullable, midnight-only timestamp column failed the read. The rule now casts the days constant with the nullability of the constant that it replaces. --------- Signed-off-by: Robert Kruszewski <[email protected]>
1 parent 52a1d15 commit e8974d6

1 file changed

Lines changed: 38 additions & 2 deletions

File tree

  • encodings/datetime-parts/src/compute

‎encodings/datetime-parts/src/compute/rules.rs‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,11 @@ impl ArrayParentReduceRule<DateTimeParts> for DTPComparisonPushDownRule {
127127
return Ok(None);
128128
};
129129
let len = days.len();
130-
let target_dtype = days.dtype();
130+
// Keep the nullability of the replaced constant, so the rewritten comparison has
131+
// the same result dtype as the original.
132+
let target_dtype = days.dtype().with_nullability(c.dtype().nullability());
131133
let constant = ConstantArray::new(days_value, len).into_array();
132-
new_children.push(constant.cast(target_dtype.clone())?);
134+
new_children.push(constant.cast(target_dtype)?);
133135
}
134136
}
135137

@@ -329,6 +331,40 @@ mod tests {
329331
assert_eq!(true_count(&optimized, &mut ctx), 3);
330332
}
331333

334+
/// Create a non-nullable DTP array at midnight with constant-zero seconds and subseconds.
335+
fn dtp_constant_zero_parts(days: &[i32]) -> VortexResult<ArrayRef> {
336+
let dtype = midnight_constant(0, TimeUnit::Seconds, 1)
337+
.dtype()
338+
.as_nonnullable();
339+
let len = days.len();
340+
341+
Ok(DateTimeParts::try_new(
342+
dtype,
343+
PrimitiveArray::new(Buffer::copy_from(days), Validity::NonNullable).into_array(),
344+
ConstantArray::new(0u8, len).into_array(),
345+
ConstantArray::new(0u8, len).into_array(),
346+
)?
347+
.into_array())
348+
}
349+
350+
#[test]
351+
fn test_binary_comparison_pushdown_nullable_constant() -> VortexResult<()> {
352+
let dtp = dtp_constant_zero_parts(&[0, 1, 2])?;
353+
let scalar = midnight_constant(1, TimeUnit::Seconds, 1)
354+
.as_constant()
355+
.vortex_expect("midnight_constant must be constant")
356+
.into_nullable();
357+
let constant = ConstantArray::new(scalar, dtp.len()).into_array();
358+
let comparison = Binary::try_new(dtp, constant, Operator::Lte)?.into_array();
359+
360+
let optimized = comparison.optimize()?;
361+
assert_eq!(optimized.dtype(), comparison.dtype());
362+
363+
let mut ctx = array_session().create_execution_ctx();
364+
assert_eq!(true_count(&optimized, &mut ctx), 2);
365+
Ok(())
366+
}
367+
332368
#[test]
333369
fn test_no_pushdown_non_midnight_constant() {
334370
// DTP with days [0, 1, 2] at midnight

0 commit comments

Comments
 (0)