Skip to content

Commit dafc832

Browse files
authored
Require matching dtypes for extension comparisons (#11016)
## Summary - Tracking Issue: #7683. Extension comparisons could compare raw storage on the array path while scalar execution rejected the same operands. Applies one dtype requirement across comparison paths. ## Changes Checks logical dtypes at binding and before comparison fast paths or encoding kernels run. Regression tests cover batch, row, and constant execution, encoded timestamps, and DuckDB filter pushdown through SQL logic tests. Broader documentation is split into #11099. DataFusion literal conversion is handled separately in #11067. Local pre-commit license and formatting checks passed. Tests and benchmarks were not run after the review fixes. Workspace-wide Clippy started through the pre-push hook and was stopped without a result. ## API Changes Comparison operands must have matching logical dtypes, ignoring nullability. Comparisons between an extension and its storage dtype, or between different extension dtypes, now return an error. This includes timestamps with different units or time zones. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent a606596 commit dafc832

11 files changed

Lines changed: 337 additions & 252 deletions

File tree

‎encodings/datetime-parts/src/compute/compare.rs‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use vortex_array::scalar_fn::fns::operators::CompareOperator;
1818
use vortex_array::scalar_fn::fns::operators::Operator;
1919
use vortex_array::validity::Validity;
2020
use vortex_error::VortexResult;
21+
use vortex_error::vortex_ensure;
2122
use vortex_mask::AllOr;
2223

2324
use crate::array::DateTimeParts;
@@ -31,6 +32,13 @@ impl CompareKernel for DateTimeParts {
3132
operator: CompareOperator,
3233
ctx: &mut ExecutionCtx,
3334
) -> VortexResult<Option<ArrayRef>> {
35+
vortex_ensure!(
36+
lhs.dtype().eq_ignore_nullability(rhs.dtype()),
37+
"Comparison requires equal dtypes ignoring nullability, got {} and {}",
38+
lhs.dtype(),
39+
rhs.dtype()
40+
);
41+
3442
let Some(rhs_const) = rhs.as_constant() else {
3543
return Ok(None);
3644
};
@@ -284,7 +292,7 @@ fn compare_dtp(
284292
}
285293

286294
#[cfg(test)]
287-
mod test {
295+
mod tests {
288296
use std::sync::LazyLock;
289297

290298
use rstest::rstest;
@@ -683,6 +691,37 @@ mod test {
683691
assert_matches_temporal(nullable, unit, constants)
684692
}
685693

694+
#[rstest]
695+
#[case::storage(Scalar::from(1500i64))]
696+
#[case::different_unit(Scalar::extension::<Timestamp>(
697+
TimestampOptions { unit: TimeUnit::Nanoseconds, tz: None },
698+
Scalar::from(1_500_000_000i64),
699+
))]
700+
fn test_comparison_rejects_mismatched_constant_dtype(
701+
#[case] scalar: Scalar,
702+
) -> VortexResult<()> {
703+
let mut ctx = SESSION.create_execution_ctx();
704+
let temporal = TemporalArray::new_timestamp(
705+
buffer![1600i64, 1400].into_array(),
706+
TimeUnit::Milliseconds,
707+
None,
708+
);
709+
let array = DateTimeParts::try_from_temporal(temporal, &mut ctx)?;
710+
let rhs = ConstantArray::new(scalar, array.len()).into_array();
711+
712+
assert!(
713+
<DateTimeParts as CompareKernel>::compare(
714+
array.as_view(),
715+
&rhs,
716+
CompareOperator::Lt,
717+
&mut ctx,
718+
)
719+
.is_err()
720+
);
721+
722+
Ok(())
723+
}
724+
686725
#[test]
687726
fn null_rows_stay_null_when_parts_differ() -> VortexResult<()> {
688727
let mut ctx = SESSION.create_execution_ctx();

‎vortex-array/src/arrays/datetime/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4-
#[cfg(test)]
5-
mod test;
6-
74
use std::sync::Arc;
85

96
use vortex_error::VortexError;
@@ -213,3 +210,6 @@ impl TryFrom<ExtensionArray> for TemporalData {
213210
Ok(Self { ext })
214211
}
215212
}
213+
214+
#[cfg(test)]
215+
mod tests;
Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,17 @@ use std::sync::LazyLock;
66
use rstest::rstest;
77
use vortex_buffer::buffer;
88
use vortex_error::VortexExpect;
9-
use vortex_error::VortexResult;
109
use vortex_session::VortexSession;
1110

1211
use crate::EqMode;
1312
use crate::IntoArray;
1413
use crate::VortexSessionExecute;
15-
use crate::arrays::BoolArray;
1614
use crate::arrays::PrimitiveArray;
1715
use crate::arrays::datetime::TemporalData;
1816
use crate::assert_arrays_eq;
19-
use crate::expr::gt;
20-
use crate::expr::lit;
21-
use crate::expr::root;
2217
use crate::extension::datetime::TemporalMetadata;
2318
use crate::extension::datetime::TimeUnit;
24-
use crate::extension::datetime::Timestamp;
25-
use crate::extension::datetime::TimestampOptions;
2619
use crate::hash::ArrayEq;
27-
use crate::scalar::Scalar;
2820
use crate::validity::Validity;
2921

3022
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
@@ -216,34 +208,3 @@ fn test_validity_preservation(#[case] validity: Validity) {
216208
.array_eq(&validity, EqMode::Ptr)
217209
);
218210
}
219-
220-
#[test]
221-
fn comparison_rejects_mismatched_timestamp_units() -> VortexResult<()> {
222-
let ts_array = PrimitiveArray::from_iter(vec![1704067200000i64, 1704153600000, 1704240000000])
223-
.into_array();
224-
let temporal = TemporalData::new_timestamp(ts_array, TimeUnit::Milliseconds, None);
225-
226-
let filter_expr = gt(
227-
root(),
228-
lit(Scalar::extension::<Timestamp>(
229-
TimestampOptions {
230-
unit: TimeUnit::Seconds,
231-
tz: None,
232-
},
233-
Scalar::from(1704153600i64),
234-
)),
235-
);
236-
237-
let error = temporal
238-
.into_array()
239-
.apply(&filter_expr)?
240-
.execute::<BoolArray>(&mut SESSION.create_execution_ctx())
241-
.unwrap_err();
242-
assert!(
243-
error
244-
.to_string()
245-
.contains("Cannot compare extension dtypes")
246-
);
247-
248-
Ok(())
249-
}

‎vortex-array/src/arrays/extension/compute/rules.rs‎

Lines changed: 0 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -246,67 +246,6 @@ mod tests {
246246
assert_eq!(constant.len(), 3);
247247
}
248248

249-
#[test]
250-
fn test_scalar_fn_no_pushdown_different_ext_types() {
251-
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
252-
struct TestExt2;
253-
impl ExtVTable for TestExt2 {
254-
type Metadata = EmptyMetadata;
255-
type NativeValue<'a> = &'a str;
256-
257-
#[expect(clippy::disallowed_methods, reason = "test-only id")]
258-
fn id(&self) -> ExtId {
259-
ExtId::new("test_ext_2")
260-
}
261-
262-
fn serialize_metadata(&self, _metadata: &Self::Metadata) -> VortexResult<Vec<u8>> {
263-
Ok(vec![])
264-
}
265-
266-
fn deserialize_metadata(&self, _data: &[u8]) -> VortexResult<Self::Metadata> {
267-
Ok(EmptyMetadata)
268-
}
269-
270-
fn validate_dtype(_extension_dtype: &ExtDType<Self>) -> VortexResult<()> {
271-
Ok(())
272-
}
273-
274-
fn unpack_native<'a>(
275-
_extension_dtype: &'a ExtDType<Self>,
276-
_storage_value: &'a ScalarValue,
277-
) -> VortexResult<Self::NativeValue<'a>> {
278-
Ok("")
279-
}
280-
}
281-
282-
let ext_dtype1 = ExtDType::<TestExt>::try_new(
283-
EmptyMetadata,
284-
DType::Primitive(PType::I64, Nullability::NonNullable),
285-
)
286-
.unwrap()
287-
.erased();
288-
289-
let storage = buffer![10i64, 20, 30].into_array();
290-
let ext_array = ExtensionArray::new(ext_dtype1, storage).into_array();
291-
292-
// Create constant with different extension type
293-
let const_scalar = Scalar::extension::<TestExt2>(EmptyMetadata, Scalar::from(25i64));
294-
let const_array = ConstantArray::new(const_scalar, 3).into_array();
295-
296-
let scalar_fn_array = Binary::try_new(ext_array, const_array, Operator::Lt)
297-
.unwrap()
298-
.into_array();
299-
300-
let optimized = scalar_fn_array.optimize().unwrap();
301-
302-
// The first child should still be an ExtensionArray (no pushdown happened)
303-
let scalar_fn = optimized.as_opt::<ScalarFn>().unwrap();
304-
assert!(
305-
scalar_fn.children()[0].as_opt::<Extension>().is_some(),
306-
"Expected first child to remain ExtensionArray when ext types differ"
307-
);
308-
}
309-
310249
#[test]
311250
fn test_scalar_fn_no_pushdown_non_constant_sibling() {
312251
let ext_dtype = test_ext_dtype();
@@ -331,27 +270,4 @@ mod tests {
331270
"Expected first child to remain ExtensionArray when sibling is not constant"
332271
);
333272
}
334-
335-
#[test]
336-
fn test_scalar_fn_no_pushdown_non_extension_constant() {
337-
let ext_dtype = test_ext_dtype();
338-
let storage = buffer![10i64, 20, 30].into_array();
339-
let ext_array = ExtensionArray::new(ext_dtype, storage).into_array();
340-
341-
// Create a non-extension constant (plain primitive)
342-
let const_array = ConstantArray::new(Scalar::from(25i64), 3).into_array();
343-
344-
let scalar_fn_array = Binary::try_new(ext_array, const_array, Operator::Lt)
345-
.unwrap()
346-
.into_array();
347-
348-
let optimized = scalar_fn_array.optimize().unwrap();
349-
350-
// No pushdown should happen because constant is not an extension scalar
351-
let scalar_fn = optimized.as_opt::<ScalarFn>().unwrap();
352-
assert!(
353-
scalar_fn.children()[0].as_opt::<Extension>().is_some(),
354-
"Expected first child to remain ExtensionArray when constant is not extension"
355-
);
356-
}
357273
}

0 commit comments

Comments
 (0)