@@ -9,10 +9,15 @@ use vortex_error::VortexResult;
99use vortex_error:: vortex_ensure;
1010
1111use crate :: ArrayRef ;
12+ use crate :: IntoArray ;
1213use crate :: array:: Array ;
1314use crate :: array:: ArrayParts ;
1415use crate :: array:: TypedArrayRef ;
16+ use crate :: arrays:: ConstantArray ;
17+ use crate :: arrays:: Extension ;
1518use crate :: arrays:: ScalarFn ;
19+ use crate :: arrays:: extension:: ExtensionArrayExt ;
20+ use crate :: dtype:: DType ;
1621use crate :: scalar_fn:: ScalarFnRef ;
1722
1823// ScalarFnArray has a variable number of slots (one per child)
@@ -84,13 +89,18 @@ impl<T: TypedArrayRef<ScalarFn>> ScalarFnArrayExt for T {}
8489
8590impl Array < ScalarFn > {
8691 /// Create a new ScalarFnArray from a scalar function and its children.
92+ ///
93+ /// When a child has a refinement extension dtype
94+ /// (`ExtVTable::is_refinement() == true`) and the scalar function rejects the original
95+ /// input shape, refinement children are transparently peeled one level at a time until
96+ /// the fn accepts the shape or no refinement children remain to peel.
8797 pub fn try_new (
8898 scalar_fn : ScalarFnRef ,
89- children : Vec < ArrayRef > ,
99+ mut children : Vec < ArrayRef > ,
90100 len : usize ,
91101 ) -> VortexResult < Self > {
92- let arg_dtypes : Vec < _ > = children . iter ( ) . map ( |c| c . dtype ( ) . clone ( ) ) . collect ( ) ;
93- let dtype = scalar_fn . return_dtype ( & arg_dtypes ) ? ;
102+ let dtype = peel_refinements_and_resolve_dtype ( & scalar_fn , & mut children ) ? ;
103+
94104 let data = ScalarFnData :: build ( scalar_fn. clone ( ) , children. clone ( ) , len) ?;
95105 let vtable = ScalarFn { id : scalar_fn. id ( ) } ;
96106 Ok ( unsafe {
@@ -101,3 +111,324 @@ impl Array<ScalarFn> {
101111 } )
102112 }
103113}
114+
115+ // TODO(connor): Refinement-preserving (e.g. `add(PositiveInt, PositiveInt) -> PositiveInt`) and
116+ // refinement-changing (e.g. `negate(PositiveInt) -> NegativeInt`) semantics require an
117+ // inverted-control hook like `ExtVTable::closure_result(&self, scalar_fn, arg_dtypes) ->
118+ // Option<DType>` so the refinement can inspect the fn and preserve/rewrite itself. Until then,
119+ // refinements are lost through any scalar fn whose `return_dtype` doesn't explicitly acceptthem
120+ // (the peel loop strips the refinement and operates on storage instead).
121+ /// Resolves the scalar function's return dtype against `children`, transparently peeling
122+ /// refinement-typed children when the fn doesn't accept the original shape.
123+ ///
124+ /// Why this exists:
125+ ///
126+ /// Refinement extensions are logically stricter views of a storage dtype. For example, a
127+ /// `DivisibleInt` is still represented as `u64`, and many scalar fns are semantically valid on
128+ /// the storage values even though their `return_dtype` implementation only understands `u64`.
129+ /// That means `add(DivisibleInt, DivisibleInt)` should be allowed to fall back to
130+ /// `add(u64, u64)` when the fn does not explicitly accept the refinement.
131+ ///
132+ /// Why this happens during construction rather than as a normal optimizer rewrite:
133+ ///
134+ /// Public array construction goes through `ScalarFnFactoryExt::try_new_array`, which must call
135+ /// `return_dtype` before a `ScalarFnArray` exists. A post-construction reduce rule would fire too
136+ /// late. So refinement peeling has to be a construction-time fallback for dtype resolution.
137+ ///
138+ /// Why peeling is intentionally narrow:
139+ ///
140+ /// Only arrays that are *representation wrappers* can be peeled safely. Today that means:
141+ /// - `ExtensionArray`, which can peel to `storage_array()`
142+ /// - `ConstantArray` holding an extension scalar, which can peel to a constant storage scalar
143+ ///
144+ /// Other encodings may preserve a refinement dtype without exposing storage as child slot 0. For
145+ /// example, `mask(refined, mask)` still has a refinement dtype, but child 0 is the masked input
146+ /// expression, not "the refinement storage". Peeling such arrays structurally via `nth_child(0)`
147+ /// silently drops semantics and can produce wrong results. In those cases we must reject
148+ /// construction until we have a semantics-preserving rewrite.
149+ ///
150+ /// The algorithm is a fixpoint:
151+ ///
152+ /// 1. Ask the scalar fn for its return dtype on the current children's dtypes.
153+ /// 2. If it succeeds, return the result. This covers both the all-non-refinement case and the case
154+ /// where the fn explicitly accepts a refinement type as a child.
155+ /// 3. If it errors, try peeling one level from every child whose dtype is an extension dtype with
156+ /// [`is_refinement`] set. Extension arrays peel to their storage arrays; constant extension
157+ /// literals peel to constant storage scalars.
158+ /// 4. If no children were peeled, return the original error.
159+ /// 5. Otherwise, loop back to step 1 with the peeled children.
160+ ///
161+ /// [`is_refinement`]: crate::dtype::extension::ExtVTable::is_refinement
162+ pub ( crate ) fn peel_refinements_and_resolve_dtype (
163+ scalar_fn : & ScalarFnRef ,
164+ children : & mut [ ArrayRef ] ,
165+ ) -> VortexResult < DType > {
166+ loop {
167+ let children_dtypes: Vec < _ > = children. iter ( ) . map ( |c| c. dtype ( ) . clone ( ) ) . collect ( ) ;
168+
169+ match scalar_fn. return_dtype ( & children_dtypes) {
170+ Ok ( dtype) => return Ok ( dtype) ,
171+ Err ( err) => {
172+ let any_peeled = peel_refinement_layers ( children) ;
173+ if !any_peeled {
174+ return Err ( err) ;
175+ }
176+ }
177+ }
178+ }
179+ }
180+
181+ // TODO(connor): Is it correct/ok to peel away refinement on all children at once? Do we need
182+ // small-step semantics here?
183+ /// Peels one layer of refinement extensions from all `children`.
184+ ///
185+ /// Returns a flag indicating whether any child was actually peeled.
186+ fn peel_refinement_layers ( children : & mut [ ArrayRef ] ) -> bool {
187+ let mut any_peeled = false ;
188+
189+ for child in children. iter_mut ( ) {
190+ if let Some ( peeled) = peel_refinement_child ( child) {
191+ * child = peeled;
192+ any_peeled = true ;
193+ }
194+ }
195+
196+ any_peeled
197+ }
198+
199+ /// Peel exactly one refinement wrapper when the array is a real representation wrapper.
200+ ///
201+ /// This must not inspect arbitrary structural children. Only `ExtensionArray` and refinement
202+ /// literals in `ConstantArray` are safe to unwrap here.
203+ fn peel_refinement_child ( child : & ArrayRef ) -> Option < ArrayRef > {
204+ let DType :: Extension ( ext_dtype) = child. dtype ( ) else {
205+ return None ;
206+ } ;
207+ if !ext_dtype. is_refinement ( ) {
208+ return None ;
209+ }
210+
211+ if let Some ( ext_array) = child. as_opt :: < Extension > ( ) {
212+ return Some ( ext_array. storage_array ( ) . clone ( ) ) ;
213+ }
214+
215+ let constant = child. as_constant ( ) ?;
216+ let ext_scalar = constant. as_extension_opt ( ) ?;
217+ Some ( ConstantArray :: new ( ext_scalar. to_storage_scalar ( ) , child. len ( ) ) . into_array ( ) )
218+ }
219+
220+ #[ cfg( test) ]
221+ mod tests {
222+ use vortex_buffer:: Buffer ;
223+ use vortex_error:: VortexResult ;
224+
225+ use super :: * ;
226+ use crate :: IntoArray ;
227+ use crate :: arrays:: ConstantArray ;
228+ use crate :: arrays:: ExtensionArray ;
229+ use crate :: arrays:: FixedSizeListArray ;
230+ use crate :: arrays:: PrimitiveArray ;
231+ use crate :: arrays:: scalar_fn:: ScalarFnArrayExt ;
232+ use crate :: builtins:: ArrayBuiltins ;
233+ use crate :: dtype:: DType ;
234+ use crate :: dtype:: Nullability ;
235+ use crate :: dtype:: PType ;
236+ use crate :: dtype:: extension:: ExtDType ;
237+ use crate :: extension:: EmptyMetadata ;
238+ use crate :: extension:: tests:: divisible_int:: DivisibleInt ;
239+ use crate :: extension:: tests:: divisible_int:: Divisor ;
240+ use crate :: extension:: tests:: even_divisible_int:: EvenDivisibleInt ;
241+ use crate :: extension:: uuid:: Uuid ;
242+ use crate :: scalar:: Scalar ;
243+ use crate :: scalar_fn:: EmptyOptions ;
244+ use crate :: scalar_fn:: ScalarFnVTableExt ;
245+ use crate :: scalar_fn:: fns:: binary:: Binary ;
246+ use crate :: scalar_fn:: fns:: mask:: Mask ;
247+ use crate :: scalar_fn:: fns:: operators:: Operator ;
248+ use crate :: validity:: Validity ;
249+
250+ fn divisible_int_dtype ( divisor : u64 ) -> VortexResult < crate :: dtype:: extension:: ExtDTypeRef > {
251+ ExtDType :: < DivisibleInt > :: try_new (
252+ Divisor ( divisor) ,
253+ DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
254+ )
255+ . map ( |dtype| dtype. erased ( ) )
256+ }
257+
258+ fn divisible_int_array ( divisor : u64 , values : Vec < u64 > ) -> VortexResult < ArrayRef > {
259+ let ext_dtype = divisible_int_dtype ( divisor) ?;
260+ let storage =
261+ PrimitiveArray :: new :: < u64 > ( Buffer :: < u64 > :: copy_from ( & values) , Validity :: NonNullable )
262+ . into_array ( ) ;
263+ Ok ( ExtensionArray :: try_new ( ext_dtype, storage) ?. into_array ( ) )
264+ }
265+
266+ fn even_divisible_int_array ( divisor : u64 , values : Vec < u64 > ) -> VortexResult < ArrayRef > {
267+ let inner_dtype = ExtDType :: < DivisibleInt > :: try_new (
268+ Divisor ( divisor) ,
269+ DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
270+ ) ?
271+ . erased ( ) ;
272+ let outer_dtype = ExtDType :: < EvenDivisibleInt > :: try_new (
273+ EmptyMetadata ,
274+ DType :: Extension ( inner_dtype. clone ( ) ) ,
275+ ) ?
276+ . erased ( ) ;
277+ let primitive =
278+ PrimitiveArray :: new :: < u64 > ( Buffer :: < u64 > :: copy_from ( & values) , Validity :: NonNullable )
279+ . into_array ( ) ;
280+ let inner_ext = ExtensionArray :: try_new ( inner_dtype, primitive) ?. into_array ( ) ;
281+ Ok ( ExtensionArray :: try_new ( outer_dtype, inner_ext) ?. into_array ( ) )
282+ }
283+
284+ fn uuid_array ( row_count : usize ) -> VortexResult < ArrayRef > {
285+ let fsl_element = DType :: Primitive ( PType :: U8 , Nullability :: NonNullable ) ;
286+ let fsl = DType :: FixedSizeList (
287+ std:: sync:: Arc :: new ( fsl_element) ,
288+ 16 ,
289+ Nullability :: NonNullable ,
290+ ) ;
291+ let uuid_dtype = ExtDType :: < Uuid > :: try_new ( Default :: default ( ) , fsl) ?. erased ( ) ;
292+ let bytes: Buffer < u8 > = Buffer :: copy_from ( vec ! [ 0u8 ; row_count * 16 ] ) ;
293+ let primitive = PrimitiveArray :: new :: < u8 > ( bytes, Validity :: NonNullable ) . into_array ( ) ;
294+ let storage = FixedSizeListArray :: try_new ( primitive, 16 , Validity :: NonNullable , row_count) ?
295+ . into_array ( ) ;
296+ Ok ( ExtensionArray :: try_new ( uuid_dtype, storage) ?. into_array ( ) )
297+ }
298+
299+ /// `Binary(Add)` applied to two `DivisibleInt` children must peel one level, producing
300+ /// a ScalarFnArray whose dtype is `Primitive(U64)`. The refinement is lost by design
301+ /// (category A).
302+ #[ test]
303+ fn peels_single_level_refinement_through_strict_add ( ) -> VortexResult < ( ) > {
304+ let lhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
305+ let rhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
306+
307+ let sfn = Binary . bind ( Operator :: Add ) ;
308+ let arr = Array :: < ScalarFn > :: try_new ( sfn, vec ! [ lhs, rhs] , 3 ) ?;
309+
310+ assert_eq ! (
311+ arr. dtype( ) ,
312+ & DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
313+ ) ;
314+ Ok ( ( ) )
315+ }
316+
317+ /// `Binary(Add)` applied to two `EvenDivisibleInt` children must peel through both
318+ /// refinement layers (EvenDivisibleInt → DivisibleInt → U64) via the fixpoint loop.
319+ #[ test]
320+ fn peels_two_level_refinement_chain_through_strict_add ( ) -> VortexResult < ( ) > {
321+ let lhs = even_divisible_int_array ( 3 , vec ! [ 0u64 , 6 , 12 ] ) ?;
322+ let rhs = even_divisible_int_array ( 3 , vec ! [ 0u64 , 6 , 12 ] ) ?;
323+
324+ let sfn = Binary . bind ( Operator :: Add ) ;
325+ let arr = Array :: < ScalarFn > :: try_new ( sfn, vec ! [ lhs, rhs] , 3 ) ?;
326+
327+ assert_eq ! (
328+ arr. dtype( ) ,
329+ & DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
330+ ) ;
331+ Ok ( ( ) )
332+ }
333+
334+ #[ test]
335+ fn peels_refinement_constant_literal_through_strict_add ( ) -> VortexResult < ( ) > {
336+ let ext_dtype = divisible_int_dtype ( 3 ) ?;
337+ let lhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
338+ let rhs = ConstantArray :: new (
339+ Scalar :: extension_ref ( ext_dtype, Scalar :: from ( 3u64 ) ) ,
340+ lhs. len ( ) ,
341+ )
342+ . into_array ( ) ;
343+
344+ let arr = Array :: < ScalarFn > :: try_new ( Binary . bind ( Operator :: Add ) , vec ! [ lhs, rhs] , 3 ) ?;
345+
346+ assert_eq ! (
347+ arr. dtype( ) ,
348+ & DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
349+ ) ;
350+ Ok ( ( ) )
351+ }
352+
353+ #[ test]
354+ fn array_builtins_binary_reuses_refinement_peeling ( ) -> VortexResult < ( ) > {
355+ let lhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
356+ let rhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
357+
358+ let arr = lhs. binary ( rhs, Operator :: Add ) ?;
359+
360+ assert_eq ! (
361+ arr. dtype( ) ,
362+ & DType :: Primitive ( PType :: U64 , Nullability :: NonNullable ) ,
363+ ) ;
364+ Ok ( ( ) )
365+ }
366+
367+ /// Regression for an easy-to-make `nth_child(0)` peel bug.
368+ ///
369+ /// `mask(refined, false)` preserves the refinement dtype, but it is not an `ExtensionArray`.
370+ /// Peeling it via child 0 would discard the `mask(...)` node entirely, after which the retry
371+ /// loop would eventually type-check `add(lhs_storage, rhs_storage)` and silently change the
372+ /// program. The correct behavior today is to reject the construction.
373+ #[ test]
374+ fn does_not_drop_masked_refinement_via_child_zero_peel ( ) -> VortexResult < ( ) > {
375+ let lhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
376+ let rhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
377+ let mask = ConstantArray :: new ( false , lhs. len ( ) ) . into_array ( ) ;
378+ let masked_lhs =
379+ Array :: < ScalarFn > :: try_new ( Mask . bind ( EmptyOptions ) , vec ! [ lhs, mask] , 3 ) ?. into_array ( ) ;
380+
381+ let result =
382+ Array :: < ScalarFn > :: try_new ( Binary . bind ( Operator :: Add ) , vec ! [ masked_lhs, rhs] , 3 ) ;
383+
384+ assert ! (
385+ result. is_err( ) ,
386+ "non-Extension refinement producers must not be peeled via structural slot access"
387+ ) ;
388+ Ok ( ( ) )
389+ }
390+
391+ /// `Uuid` is a non-refinement extension (`is_refinement() == false`). When fed to a
392+ /// strict primitive-typed scalar fn, peeling must NOT happen — the fn's original error
393+ /// is surfaced as-is.
394+ #[ test]
395+ fn does_not_peel_non_refinement_extension ( ) -> VortexResult < ( ) > {
396+ let lhs = uuid_array ( 2 ) ?;
397+ let rhs = uuid_array ( 2 ) ?;
398+
399+ let sfn = Binary . bind ( Operator :: Add ) ;
400+ let result = Array :: < ScalarFn > :: try_new ( sfn, vec ! [ lhs, rhs] , 2 ) ;
401+
402+ assert ! (
403+ result. is_err( ) ,
404+ "Uuid is not a refinement; peel must not fire"
405+ ) ;
406+ Ok ( ( ) )
407+ }
408+
409+ /// When the scalar fn's `return_dtype` already accepts the refinement (category B / C /
410+ /// D — specialization path), the peel loop must short-circuit without touching
411+ /// children.
412+ #[ test]
413+ fn does_not_peel_when_scalar_fn_accepts_refinement ( ) -> VortexResult < ( ) > {
414+ // `Binary(Eq)` is a comparison, and its return_dtype accepts any pair of matching
415+ // extension dtypes (see `binary::mod::return_dtype` — comparisons allow extensions
416+ // as long as the two sides share the same dtype). So `Eq(DivisibleInt,
417+ // DivisibleInt)` succeeds at return_dtype time with no peel.
418+ let lhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
419+ let rhs = divisible_int_array ( 3 , vec ! [ 0u64 , 3 , 6 ] ) ?;
420+
421+ let sfn = Binary . bind ( Operator :: Eq ) ;
422+ let arr = Array :: < ScalarFn > :: try_new ( sfn, vec ! [ lhs, rhs] , 3 ) ?;
423+
424+ // Comparison returns Bool; children retain their refinement dtypes.
425+ assert_eq ! ( arr. dtype( ) , & DType :: Bool ( Nullability :: NonNullable ) ) ;
426+ let child0 = arr. child_at ( 0 ) ;
427+ assert ! (
428+ matches!( child0. dtype( ) , DType :: Extension ( ext) if ext. is:: <DivisibleInt >( ) ) ,
429+ "child 0 retained its DivisibleInt refinement (got {})" ,
430+ child0. dtype( ) ,
431+ ) ;
432+ Ok ( ( ) )
433+ }
434+ }
0 commit comments