Skip to content

Commit 4f63e7c

Browse files
connortsui20claude
andcommitted
Port the numeric binary kernels to RowFn
Progress towards #9128. `vortex.numeric` becomes a `RowFn` over a primitive pair, taking `NumericOperator` as its options, which deletes the hand-written null propagation, constant folding, and validity logic in `numeric/primitive.rs`. Checked arithmetic reports overflow as evidence the row closure returns, rather than as a comparison the caller re-derives. Decimal keeps its own columnar implementation. `PrimitiveOperand` was defined in `numeric/primitive.rs` and shared out of `numeric/mod.rs`. The port drops its numeric caller, so it moves into `compare/primitive.rs`, its only remaining user. `NumericOperator` gains `Hash`, which a `RowFn`'s options require. `map_checked_into` in `vortex-compute` loses its last caller with the port and is deleted. Also adds a `list_length` test pinning that a non-nullable fixed-size list keeps a constant result rather than materializing one `u64` per row, which is the reason `vortex.list.length` stays on `ScalarFnVTable`. Signed-off-by: Connor Tsui <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent 9fe30c4 commit 4f63e7c

9 files changed

Lines changed: 456 additions & 471 deletions

File tree

‎vortex-array/src/scalar/typed_view/primitive/numeric_operator.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use vortex_error::vortex_err;
1010

1111
use crate::scalar_fn::fns::operators::Operator;
1212

13-
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1414
/// Binary element-wise operations.
1515
pub enum NumericOperator {
1616
/// Binary element-wise addition of two arrays or of two scalars.

‎vortex-array/src/scalar_fn/fns/binary/compare/primitive.rs‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,28 @@
44
//! Native comparison of primitive arrays via bit-packing lane kernels.
55
66
use vortex_buffer::BitBuffer;
7+
use vortex_buffer::Buffer;
78
use vortex_error::VortexResult;
89
use vortex_error::vortex_bail;
910

1011
use crate::ArrayRef;
1112
use crate::ExecutionCtx;
1213
use crate::IntoArray;
1314
use crate::arrays::BoolArray;
15+
use crate::arrays::Constant;
1416
use crate::arrays::ConstantArray;
17+
use crate::arrays::PrimitiveArray;
1518
use crate::dtype::DType;
1619
use crate::dtype::NativePType;
1720
use crate::dtype::Nullability;
1821
use crate::dtype::PType;
1922
use crate::match_each_native_ptype;
2023
use crate::scalar::Scalar;
21-
use crate::scalar_fn::fns::binary::PrimitiveOperand;
2224
use crate::scalar_fn::fns::binary::compare::collect_bits;
2325
use crate::scalar_fn::fns::binary::compare::collect_zip_bits;
2426
use crate::scalar_fn::fns::binary::compare::compare_validity;
2527
use crate::scalar_fn::fns::operators::CompareOperator;
28+
use crate::validity::Validity;
2629

2730
/// Compare two primitive arrays of the same [`PType`].
2831
///
@@ -128,3 +131,67 @@ fn compare_slice_constant<T: NativePType>(lhs: &[T], rhs: T, op: CompareOperator
128131
CompareOperator::Lte => collect_bits(lhs, |a: T| a.is_le(rhs)),
129132
}
130133
}
134+
135+
/// A primitive binary-operator operand: a materialized buffer, a non-null constant, or an
136+
/// all-null constant.
137+
///
138+
/// Splitting the constant out of the buffer is what lets the lane kernels above hoist it into a
139+
/// register instead of reading it back per lane.
140+
enum PrimitiveOperand<T: NativePType> {
141+
/// A decoded column, one value per row.
142+
Array {
143+
values: Buffer<T>,
144+
validity: Validity,
145+
},
146+
147+
/// The same non-null value in every row.
148+
Constant {
149+
value: T,
150+
len: usize,
151+
validity: Validity,
152+
},
153+
154+
/// A null in every row, carrying only the row count.
155+
Null(usize),
156+
}
157+
158+
impl<T: NativePType> PrimitiveOperand<T> {
159+
fn try_new(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self> {
160+
if let Some(constant) = array.as_opt::<Constant>() {
161+
return Ok(
162+
match constant.scalar().as_primitive().try_typed_value::<T>()? {
163+
Some(value) => Self::Constant {
164+
value,
165+
len: array.len(),
166+
validity: if constant.scalar().dtype().is_nullable() {
167+
Validity::AllValid
168+
} else {
169+
Validity::NonNullable
170+
},
171+
},
172+
None => Self::Null(array.len()),
173+
},
174+
);
175+
}
176+
177+
let array = array.clone().execute::<PrimitiveArray>(ctx)?;
178+
let validity = array.validity()?;
179+
let values = array.into_buffer::<T>();
180+
Ok(Self::Array { values, validity })
181+
}
182+
183+
fn len(&self) -> usize {
184+
match self {
185+
Self::Array { values, .. } => values.len(),
186+
Self::Constant { len, .. } | Self::Null(len) => *len,
187+
}
188+
}
189+
190+
fn validity(&self) -> Validity {
191+
match self {
192+
Self::Array { validity, .. } => validity.clone(),
193+
Self::Constant { validity, .. } => validity.clone(),
194+
Self::Null(_) => Validity::AllInvalid,
195+
}
196+
}
197+
}
Lines changed: 15 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4-
//! Checked-lane execution for numeric kernels, driven by the shared
4+
//! Checked-lane execution for the decimal kernels, driven by the shared
55
//! `vortex-compute` lane kernels.
6-
7-
use std::ops::BitOrAssign;
6+
//!
7+
//! The primitive widths do not come through here: they are computed one row at a time by
8+
//! [`row`](super::row), which writes a value for every row and reduces failure as one bit rather
9+
//! than scanning lanes.
810
911
use vortex_buffer::Buffer;
1012
use vortex_buffer::BufferMut;
@@ -13,34 +15,22 @@ use vortex_compute::lane_kernels::IndexedSourceExt;
1315
use vortex_mask::AllOr;
1416
use vortex_mask::Mask;
1517

16-
/// Evidence that a lane failed, anything other than [`Default`] meaning failure.
17-
///
18-
/// `bool` is the ordinary choice; [`map_checked_into`] explains why the wider members exist and
19-
/// asserts the width bound that membership here does **not** imply.
20-
///
21-
/// [`map_checked_into`]: IndexedSourceExt::map_checked_into
22-
pub(super) trait Failure: Copy + Default + PartialEq + BitOrAssign {}
23-
24-
impl Failure for bool {}
25-
impl Failure for u8 {}
26-
impl Failure for u16 {}
27-
impl Failure for u32 {}
28-
impl Failure for u64 {}
29-
3018
/// Apply the fallible `apply` over every lane of `source`, returning
31-
/// `Err(first_failing_valid_lane)` only when it returns `None` on a _valid_ lane.
19+
/// `Err(first_failing_valid_lane)` only when it returns `None` on a valid lane.
3220
///
3321
/// `apply` also runs on invalid lanes, whose failures are masked out and whose values are
3422
/// unspecified, so it must be total: no panics or side effects on any stored lane value.
3523
///
36-
/// This drives the one-pass early-exit kernels, which abort at the end of the enclosing 64-lane
37-
/// chunk. Use it when per-lane failure handling is cheap relative to the operation, as in integer
38-
/// division. Prefer [`checked_apply_lanes`] when the failure check itself vectorizes.
24+
/// This drives the one-pass early-exit kernels: failures abort at the end of the enclosing
25+
/// 64-lane chunk. It suits an operation whose per-lane failure handling is cheap relative to the
26+
/// operation itself, which is what the decimal kernels and their per-lane casts are.
3927
///
40-
/// `#[inline]`: this must inline into the caller that builds the closure, so that a captured
41-
/// constant operand flattens into a register rather than living behind a pointer the loop reloads
42-
/// on every lane, which blocks vectorization.
43-
#[inline]
28+
/// `#[inline(always)]`: this wrapper and its kernel calls must inline into the caller that
29+
/// constructs the closure, so the closure environment (e.g. a captured constant operand)
30+
/// flattens into registers. Left to its own devices under `codegen-units > 1`, the compiler
31+
/// keeps the environment behind a pointer, and reloading a captured constant on every lane
32+
/// blocks vectorization of the whole loop.
33+
#[inline(always)]
4434
pub(super) fn checked_lanes<S, T, Apply>(
4535
source: S,
4636
valid_rows: &Mask,
@@ -61,7 +51,6 @@ where
6151
};
6252

6353
let mut values = BufferMut::<T>::with_capacity(len);
64-
6554
let out = &mut values.spare_capacity_mut()[..len];
6655
match valid_bits {
6756
None => source.try_map_into(out, apply)?,
@@ -73,55 +62,3 @@ where
7362

7463
Ok(values.freeze())
7564
}
76-
77-
/// Apply the split value/failure `apply` over every lane of `source`, returning
78-
/// `Err(first_failing_valid_lane)` only when it flags a _valid_ lane.
79-
///
80-
/// The hot pass writes every value unconditionally and OR-reduces one piece of evidence, leaving
81-
/// the loop free of per-lane selects and per-chunk exit branches. Only if a lane flagged does a
82-
/// cold second pass re-run `apply` through the early-exit kernels, which drop null-lane failures
83-
/// and attribute the first valid one.
84-
///
85-
/// Like [`checked_lanes`], `apply` runs on invalid lanes and must be total.
86-
///
87-
/// `#[inline]`: see [`checked_lanes`].
88-
#[inline]
89-
pub(super) fn checked_apply_lanes<S, T, Fail, Apply>(
90-
source: S,
91-
valid_rows: &Mask,
92-
mut apply: Apply,
93-
) -> Result<Buffer<T>, usize>
94-
where
95-
S: IndexedSource + Copy,
96-
T: Copy + Default,
97-
Fail: Failure,
98-
Apply: FnMut(S::Item) -> (T, Fail),
99-
{
100-
let len = source.len();
101-
debug_assert_eq!(len, valid_rows.len());
102-
103-
let valid_bits = match valid_rows.bit_buffer() {
104-
AllOr::All => None,
105-
AllOr::None => return Ok(Buffer::zeroed(len)),
106-
AllOr::Some(valid_bits) => Some(valid_bits),
107-
};
108-
109-
let mut values = BufferMut::<T>::with_capacity(len);
110-
111-
let out = &mut values.spare_capacity_mut()[..len];
112-
if source.map_checked_into(out, &mut apply) != Fail::default() {
113-
let mut checked = |item: S::Item| {
114-
let (value, failure) = apply(item);
115-
(failure == Fail::default()).then_some(value)
116-
};
117-
match valid_bits {
118-
None => source.try_map_into(out, &mut checked)?,
119-
Some(valid_bits) => source.try_map_masked_into(valid_bits, out, &mut checked)?,
120-
}
121-
}
122-
123-
// SAFETY: the kernels initialize every lane in `out`.
124-
unsafe { values.set_len(len) };
125-
126-
Ok(values.freeze())
127-
}

‎vortex-array/src/scalar_fn/fns/binary/numeric/mod.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,21 @@
44
//! Native execution of the arithmetic operators (Add/Sub/Mul/Div) of the [`Binary`] scalar
55
//! function. There is no Arrow fallback.
66
//!
7+
//! The primitive widths are computed by a [`RowFn`](crate::scalar_fn::RowFn), which owns null
8+
//! handling, constants and validity for them; see [`row`]. Decimal keeps its own columnar
9+
//! implementation in [`decimal`].
10+
//!
711
//! [`Binary`]: super::Binary
812
913
mod checked;
1014
mod decimal;
1115
mod primitive;
16+
mod row;
1217
#[cfg(test)]
1318
mod tests;
1419

1520
use decimal::execute_numeric_decimal;
16-
pub(crate) use primitive::PrimitiveOperand;
17-
use primitive::execute_numeric_primitive;
21+
use row::execute_numeric_primitive;
1822
use vortex_error::VortexResult;
1923
use vortex_error::vortex_ensure;
2024

0 commit comments

Comments
 (0)