Skip to content

Commit 26d881b

Browse files
committed
feat(array): replace extension cast hooks with a session cast registry
Cast rules on `ExtVTable` tied each cast to the dtype that defines it, so nothing outside the extension type could add or change a cast, and casts between built-in dtypes were not pluggable at all. `CastSession` now owns an ordered `CastRules` registry. A `CastRule` binds a `(source, target)` pair to a `CastFn` that runs over any array, and the executor consults the rules before every built-in cast, both in `Cast::execute` and in the fused `CastExecuteAdaptor` kernels. The most recently registered rule wins, so a rule can add a cast Vortex does not perform itself or replace a built-in one. `CastSession::default` installs `TimestampCast`, the unit conversion that used to live on the timestamp vtable; `CastSession::empty` installs none. Because the rules live in the session, typing a cast cannot consult them: `Cast::return_dtype` accepts every pair again and an unsupported cast fails when it executes. The built-in matrix is kept as `is_builtin_cast` so `Scalar::cast` and constant folding know which casts need no rule. The optimizer, which has no session, folds constants and literals with the built-in casts only; rule-only casts are left in place and reach the rule. `Scalar::cast` runs built-in casts directly and falls back to the default session's rules for the rest. The `ext_wrap` scalar function is no longer needed and is removed. Refs #8570 Signed-off-by: Claude <[email protected]> Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_012GiQdVvEQUshNCyjNFQUQL
1 parent 4027b71 commit 26d881b

23 files changed

Lines changed: 646 additions & 764 deletions

File tree

‎vortex-array/proto/expr.proto‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,6 @@ message CastOpts {
106106
vortex.dtype.DType target = 1;
107107
}
108108

109-
message ExtWrapOpts {
110-
vortex.dtype.DType ext_dtype = 1;
111-
}
112-
113109
message FieldNames {
114110
repeated string names = 1;
115111
}

‎vortex-array/src/arrays/constant/compute/cast.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@ use crate::dtype::DType;
1212
use crate::scalar_fn::fns::cast::CastReduce;
1313

1414
impl CastReduce for Constant {
15+
/// Folds the built-in casts. Session cast rules are consulted when the cast executes, where
16+
/// the session is known, and a failing cast is left in place so its error surfaces there.
1517
fn cast(array: ArrayView<'_, Constant>, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
16-
match array.scalar().cast(dtype) {
17-
Ok(scalar) => Ok(Some(ConstantArray::new(scalar, array.len()).into_array())),
18-
Err(_) => Ok(None),
18+
match array.scalar().cast_builtin(dtype) {
19+
Ok(Some(scalar)) => Ok(Some(ConstantArray::new(scalar, array.len()).into_array())),
20+
Ok(None) | Err(_) => Ok(None),
1921
}
2022
}
2123
}

‎vortex-array/src/arrays/extension/compute/cast.rs‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ use crate::scalar_fn::fns::cast::CastReduce;
1616
impl CastReduce for Extension {
1717
/// Handles the two extension casts that need no extension-specific knowledge: unwrapping to
1818
/// the storage dtype, and changing the nullability of the same extension dtype. Every other
19-
/// extension cast is a rewrite chosen by [`Cast::plan`](crate::scalar_fn::fns::cast::Cast)
20-
/// and never reaches this rule.
19+
/// extension cast comes from a session [`CastRule`](crate::scalar_fn::fns::cast::CastRule),
20+
/// which the executor consults before this rule.
2121
fn cast(array: ArrayView<'_, Extension>, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
2222
let ext_dtype = array.ext_dtype();
2323

@@ -52,6 +52,7 @@ mod tests {
5252
use vortex_session::VortexSession;
5353

5454
use super::*;
55+
use crate::ArrayRef;
5556
use crate::IntoArray;
5657
use crate::arrays::ConstantArray;
5758
use crate::arrays::PrimitiveArray;
@@ -65,6 +66,7 @@ mod tests {
6566
use crate::extension::datetime::TimeUnit;
6667
use crate::extension::datetime::Timestamp;
6768
use crate::scalar::Scalar;
69+
use crate::scalar_fn::fns::cast::CastSession;
6870

6971
static SESSION: LazyLock<VortexSession> = LazyLock::new(crate::array_session);
7072

@@ -176,7 +178,7 @@ mod tests {
176178
Scalar::from(3i64),
177179
);
178180

179-
// The same rewrite serves the scalar and the constant array.
181+
// The same rule serves the scalar and the constant array.
180182
let expected = Scalar::extension_ref(
181183
target.as_extension_opt().vortex_expect("extension").clone(),
182184
Scalar::from(3_000i64),
@@ -191,46 +193,50 @@ mod tests {
191193
Ok(())
192194
}
193195

194-
#[test]
195-
fn cast_timestamp_rejects_timezone_change() {
196-
let source = timestamp_dtype(
197-
TimeUnit::Milliseconds,
198-
Some("UTC"),
199-
Nullability::NonNullable,
200-
);
201-
let target = timestamp_dtype(
202-
TimeUnit::Nanoseconds,
203-
Some("Europe/London"),
204-
Nullability::NonNullable,
205-
);
206-
let arr = ExtensionArray::new(
207-
source.as_extension_opt().vortex_expect("extension").clone(),
196+
fn timestamp_array(dtype: &DType) -> ArrayRef {
197+
ExtensionArray::new(
198+
dtype.as_extension_opt().vortex_expect("extension").clone(),
208199
buffer![1i64].into_array(),
209200
)
210-
.into_array();
211-
212-
// Rejected when the cast is bound, before any execution.
213-
let result = arr.cast(target);
214-
assert!(result.is_err(), "expected error, got {result:?}");
201+
.into_array()
215202
}
216203

217-
#[test]
218-
fn cast_storage_to_extension_requires_opt_in() {
219-
let target = timestamp_dtype(TimeUnit::Milliseconds, None, Nullability::NonNullable);
220-
let result = buffer![1i64].into_array().cast(target);
204+
/// Unsupported casts bind, because the rules live in the session, and fail at execution.
205+
#[rstest]
206+
#[case::timezone_change(
207+
timestamp_array(&timestamp_dtype(TimeUnit::Milliseconds, Some("UTC"), Nullability::NonNullable)),
208+
timestamp_dtype(TimeUnit::Nanoseconds, Some("Europe/London"), Nullability::NonNullable)
209+
)]
210+
#[case::storage_to_extension_requires_a_rule(
211+
buffer![1i64].into_array(),
212+
timestamp_dtype(TimeUnit::Milliseconds, None, Nullability::NonNullable)
213+
)]
214+
#[case::wider_than_storage(
215+
timestamp_array(&timestamp_dtype(TimeUnit::Milliseconds, None, Nullability::NonNullable)),
216+
DType::Primitive(PType::F64, Nullability::NonNullable)
217+
)]
218+
fn unsupported_casts_fail_at_execution(#[case] array: ArrayRef, #[case] target: DType) {
219+
let mut ctx = SESSION.create_execution_ctx();
220+
let cast = array.cast(target).vortex_expect("casts always bind");
221+
let result = cast.execute::<ArrayRef>(&mut ctx);
221222
assert!(result.is_err(), "expected error, got {result:?}");
222223
}
223224

224225
#[test]
225-
fn cast_extension_to_wider_storage_is_rejected() {
226-
let source = timestamp_dtype(TimeUnit::Milliseconds, None, Nullability::NonNullable);
227-
let arr = ExtensionArray::new(
228-
source.as_extension_opt().vortex_expect("extension").clone(),
229-
buffer![1i64].into_array(),
230-
)
231-
.into_array();
226+
fn cast_timestamp_unit_needs_the_rule() {
227+
let session = VortexSession::empty().with_some(CastSession::empty());
228+
let mut ctx = session.create_execution_ctx();
229+
let source = timestamp_dtype(TimeUnit::Seconds, Some("UTC"), Nullability::NonNullable);
230+
let target = timestamp_dtype(
231+
TimeUnit::Milliseconds,
232+
Some("UTC"),
233+
Nullability::NonNullable,
234+
);
232235

233-
let result = arr.cast(DType::Primitive(PType::F64, Nullability::NonNullable));
236+
let result = timestamp_array(&source)
237+
.cast(target)
238+
.vortex_expect("casts always bind")
239+
.execute::<ArrayRef>(&mut ctx);
234240
assert!(result.is_err(), "expected error, got {result:?}");
235241
}
236242

‎vortex-array/src/arrays/map/tests.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use vortex_session::registry::ReadContext;
1010
use crate::Array;
1111
use crate::ArrayContext;
1212
use crate::ArrayParts;
13+
use crate::ArrayRef;
1314
use crate::ArrayVTable;
1415
use crate::Canonical;
1516
use crate::IntoArray;
@@ -618,11 +619,13 @@ fn null_map_cast_cannot_create_sortedness_assertion() -> VortexResult<()> {
618619
let scalar = Scalar::null(unsorted_dtype);
619620

620621
assert!(scalar.cast(&sorted_dtype).is_err());
621-
// The cast is rejected when it is bound, before any execution.
622+
// The cast binds and fails when it executes.
623+
let mut ctx = array_session().create_execution_ctx();
622624
assert!(
623625
ConstantArray::new(scalar, 2)
624626
.into_array()
625627
.cast(sorted_dtype.clone())
628+
.and_then(|array| array.execute::<ArrayRef>(&mut ctx))
626629
.is_err()
627630
);
628631

‎vortex-array/src/builtins.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ impl ArrayBuiltins for ArrayRef {
172172
if self.dtype() == &dtype {
173173
return Ok(self.clone());
174174
}
175-
Cast::try_new(self.clone(), dtype)?.into_array().optimize()
175+
Cast::new(self.clone(), dtype).into_array().optimize()
176176
}
177177

178178
fn fill_null(&self, fill_value: impl Into<Scalar>) -> VortexResult<ArrayRef> {

‎vortex-array/src/dtype/extension/erased.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ use crate::dtype::extension::ExtId;
2020
use crate::dtype::extension::ExtVTable;
2121
use crate::dtype::extension::Matcher;
2222
use crate::dtype::extension::typed::DynExtDType;
23-
use crate::expr::Expression;
2423
use crate::scalar::ScalarValue;
2524

2625
/// A type-erased extension dtype.
@@ -108,18 +107,6 @@ impl ExtDTypeRef {
108107
pub(crate) fn validate_storage_value(&self, storage_value: &ScalarValue) -> VortexResult<()> {
109108
self.0.validate_scalar_value(storage_value)
110109
}
111-
112-
/// Returns an expression casting this extension dtype to `target`, if the extension type
113-
/// supports it. See [`ExtVTable::cast_to`].
114-
pub fn cast_to(&self, target: &DType) -> VortexResult<Option<Expression>> {
115-
self.0.cast_to(target)
116-
}
117-
118-
/// Returns an expression casting `source` to this extension dtype, if the extension type
119-
/// supports it. See [`ExtVTable::cast_from`].
120-
pub fn cast_from(&self, source: &DType) -> VortexResult<Option<Expression>> {
121-
self.0.cast_from(source)
122-
}
123110
}
124111

125112
/// Methods for downcasting type-erased extension dtypes.

‎vortex-array/src/dtype/extension/typed.rs‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ use crate::dtype::Nullability;
2020
use crate::dtype::extension::ExtDTypeRef;
2121
use crate::dtype::extension::ExtId;
2222
use crate::dtype::extension::ExtVTable;
23-
use crate::expr::Expression;
2423
use crate::scalar::ScalarValue;
2524

2625
/// A typed extension data type, parameterized by a concrete [`ExtVTable`].
@@ -109,18 +108,6 @@ impl<V: ExtVTable> ExtDType<V> {
109108
V::validate_scalar_value(self, storage_value)
110109
}
111110

112-
/// Returns an expression casting this extension dtype to `target`, if the extension type
113-
/// supports it. See [`ExtVTable::cast_to`].
114-
pub fn cast_to(&self, target: &DType) -> VortexResult<Option<Expression>> {
115-
V::cast_to(self, target)
116-
}
117-
118-
/// Returns an expression casting `source` to this extension dtype, if the extension type
119-
/// supports it. See [`ExtVTable::cast_from`].
120-
pub fn cast_from(&self, source: &DType) -> VortexResult<Option<Expression>> {
121-
V::cast_from(self, source)
122-
}
123-
124111
/// Erase the concrete type information, returning a type-erased extension dtype.
125112
pub fn erased(self) -> ExtDTypeRef {
126113
ExtDTypeRef(Arc::new(self))
@@ -146,8 +133,6 @@ pub(super) trait DynExtDType: 'static + Send + Sync + super::sealed::Sealed {
146133
fn validate_scalar_value(&self, storage_value: &ScalarValue) -> VortexResult<()>;
147134
fn value_display(&self, f: &mut fmt::Formatter<'_>, storage_value: &ScalarValue)
148135
-> fmt::Result;
149-
fn cast_to(&self, target: &DType) -> VortexResult<Option<Expression>>;
150-
fn cast_from(&self, source: &DType) -> VortexResult<Option<Expression>>;
151136
}
152137

153138
/// Blanket impl: thin forwarder to `ExtDType<V>` inherent methods.
@@ -202,14 +187,6 @@ impl<V: ExtVTable> DynExtDType for ExtDType<V> {
202187
self.validate_scalar_value(storage_value)
203188
}
204189

205-
fn cast_to(&self, target: &DType) -> VortexResult<Option<Expression>> {
206-
self.cast_to(target)
207-
}
208-
209-
fn cast_from(&self, source: &DType) -> VortexResult<Option<Expression>> {
210-
self.cast_from(source)
211-
}
212-
213190
fn value_display(
214191
&self,
215192
f: &mut fmt::Formatter<'_>,

‎vortex-array/src/dtype/extension/vtable.rs‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@ use std::hash::Hash;
77

88
use vortex_error::VortexResult;
99

10-
use crate::dtype::DType;
1110
use crate::dtype::extension::ExtDType;
1211
use crate::dtype::extension::ExtId;
13-
use crate::expr::Expression;
1412
use crate::scalar::ScalarValue;
1513

1614
/// The public API for defining new extension types.
@@ -50,44 +48,6 @@ pub trait ExtVTable: 'static + Sized + Send + Sync + Clone + Debug + Eq + Hash {
5048
/// extension metadata.
5149
fn validate_dtype(ext_dtype: &ExtDType<Self>) -> VortexResult<()>;
5250

53-
// Methods related to casting.
54-
55-
/// Returns an expression that casts a value of this extension dtype to `target`.
56-
///
57-
/// The expression is written over [`root()`](crate::expr::root), which stands for the input
58-
/// value, and must evaluate to exactly `target`. Because it is a plain expression, the same
59-
/// definition serves arrays, constants, and scalars, and it takes part in expression
60-
/// optimization like any other cast. Building it from the concrete [`ExtDType`] lets the
61-
/// implementation consult extension metadata, such as a timestamp's unit and timezone.
62-
///
63-
/// Returning `Ok(None)` declines the cast. The default cast rules then apply: an extension
64-
/// dtype can always be cast to its storage dtype, and a `target` extension dtype gets a
65-
/// chance to accept the cast through [`cast_from`](Self::cast_from). Anything else is
66-
/// rejected when the cast is type-checked, so a cast that is accepted here can be
67-
/// planned without executing it.
68-
///
69-
/// Identity and nullability-only casts never reach this hook.
70-
fn cast_to(ext_dtype: &ExtDType<Self>, target: &DType) -> VortexResult<Option<Expression>> {
71-
_ = (ext_dtype, target);
72-
Ok(None)
73-
}
74-
75-
/// Returns an expression that casts a value of `source` to this extension dtype.
76-
///
77-
/// The expression follows the same contract as [`cast_to`](Self::cast_to): it is written
78-
/// over [`root()`](crate::expr::root) and must evaluate to exactly this extension dtype.
79-
/// [`ext_wrap`](crate::expr::ext_wrap) wraps a storage-typed expression into an extension
80-
/// dtype.
81-
///
82-
/// There is no default cast into an extension dtype, not even from its storage dtype,
83-
/// because the storage dtype alone does not prove that values are meaningful for the
84-
/// extension type. Extension types opt in through this hook. When `source` is itself an
85-
/// extension dtype, its own [`cast_to`](Self::cast_to) is consulted first.
86-
fn cast_from(ext_dtype: &ExtDType<Self>, source: &DType) -> VortexResult<Option<Expression>> {
87-
_ = (ext_dtype, source);
88-
Ok(None)
89-
}
90-
9151
// Methods related to the extension scalar values.
9252

9353
/// Validate the given storage value is compatible with the extension type.

‎vortex-array/src/expr/exprs.rs‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ use crate::dtype::DType;
1515
use crate::dtype::FieldName;
1616
use crate::dtype::FieldNames;
1717
use crate::dtype::Nullability;
18-
use crate::dtype::extension::ExtDTypeRef;
1918
use crate::expr::BoundExpression;
2019
use crate::expr::Expression;
2120
use crate::scalar::Scalar;
@@ -33,7 +32,6 @@ use crate::scalar_fn::fns::dynamic::DynamicComparison;
3332
use crate::scalar_fn::fns::dynamic::DynamicComparisonExpr;
3433
use crate::scalar_fn::fns::dynamic::Rhs;
3534
use crate::scalar_fn::fns::ext_storage::ExtStorage;
36-
use crate::scalar_fn::fns::ext_wrap::ExtWrap;
3735
use crate::scalar_fn::fns::fill_null::FillNull;
3836
use crate::scalar_fn::fns::get_item::GetItem;
3937
use crate::scalar_fn::fns::is_not_null::IsNotNull;
@@ -1157,30 +1155,6 @@ pub fn bound_ext_storage(input: BoundExpression) -> BoundExpression {
11571155
.vortex_expect("extension-storage expressions require an extension child")
11581156
}
11591157

1160-
// ---- ExtWrap ----
1161-
1162-
/// Creates an expression that wraps storage values into an extension dtype.
1163-
///
1164-
/// The input must have exactly the storage dtype of `ext_dtype`. This is the inverse of
1165-
/// [`ext_storage`] and the building block for extension cast rewrites.
1166-
///
1167-
/// ```rust
1168-
/// # use vortex_array::dtype::Nullability;
1169-
/// # use vortex_array::expr::{ext_wrap, root};
1170-
/// # use vortex_array::extension::datetime::{TimeUnit, Timestamp};
1171-
/// let expr = ext_wrap(root(), Timestamp::new(TimeUnit::Seconds, Nullability::NonNullable).erased());
1172-
/// ```
1173-
pub fn ext_wrap(input: Expression, ext_dtype: ExtDTypeRef) -> Expression {
1174-
ExtWrap.new_expr(ext_dtype, [input])
1175-
}
1176-
1177-
/// Creates a bound expression that wraps storage values into an extension dtype.
1178-
pub fn bound_ext_wrap(input: BoundExpression, ext_dtype: ExtDTypeRef) -> BoundExpression {
1179-
ExtWrap
1180-
.try_new_bound_expr(ext_dtype, [input])
1181-
.vortex_expect("ext_wrap expressions require a child with the extension's storage dtype")
1182-
}
1183-
11841158
// ---- ListLength ----
11851159

11861160
/// Creates an expression that computes the number of elements in each list
@@ -1263,7 +1237,6 @@ pub mod bound {
12631237
pub use super::bound_dynamic_with_options as dynamic_with_options;
12641238
pub use super::bound_eq as eq;
12651239
pub use super::bound_ext_storage as ext_storage;
1266-
pub use super::bound_ext_wrap as ext_wrap;
12671240
pub use super::bound_fill_null as fill_null;
12681241
pub use super::bound_get_item as get_item;
12691242
pub use super::bound_gt as gt;

‎vortex-array/src/expr/mod.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ pub use exprs::dynamic;
8787
pub use exprs::dynamic_with_options;
8888
pub use exprs::eq;
8989
pub use exprs::ext_storage;
90-
pub use exprs::ext_wrap;
9190
pub use exprs::fill_null;
9291
pub use exprs::get_item;
9392
pub use exprs::gt;

0 commit comments

Comments
 (0)