Skip to content

Commit 1a09eb4

Browse files
authored
Execute interleaved decimal arrays (#10940)
## Summary - Tracking Issue: #10701 - CD4 (#10946) builds on this PR. `Interleave` accepts decimal values but panics when it executes them, because only the bool and primitive kernels exist. Decimal `Widen` (CD4) falls back to canonical decimals for interleave, so it needs a decimal kernel first. ## Changes Most of the diff is a refactor of the primitive kernel: its gather moves into a `fixed_width` module next to the kernels, so decimals reuse it instead of getting a second copy. Canonical decimals are fixed-width buffers just like primitives, so one set of selector bounds checks and one tuned gather loop can serve both. The gather stays in `interleave` and not in `arrays/fixed_width` because it owns the selector contract of `Interleave` and has no other caller. Decimal sources stored at a non-canonical width are copied to the canonical width before the gather. That copy goes away once CD8a stores every decimal at its canonical width. ## API Changes `Interleave` executes decimal values and returns them at the canonical width of their dtype, where it used to panic. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent 48a3365 commit 1a09eb4

5 files changed

Lines changed: 500 additions & 116 deletions

File tree

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Execution for decimal [`Interleave`] values.
5+
//!
6+
//! Sources can use different stored widths for the same logical dtype. Decimal execution
7+
//! normalizes them to the logical canonical width before gathering rows.
8+
9+
use vortex_buffer::Buffer;
10+
use vortex_error::VortexResult;
11+
use vortex_error::vortex_err;
12+
13+
use super::super::Interleave;
14+
use super::super::InterleaveArrayExt;
15+
use super::super::InterleaveSlots;
16+
use super::fixed_width::Source;
17+
use super::fixed_width::gather;
18+
use crate::AnyColumnar;
19+
use crate::ArrayRef;
20+
use crate::array::Array;
21+
use crate::arrays::Constant;
22+
use crate::arrays::Decimal;
23+
use crate::arrays::DecimalArray;
24+
use crate::arrays::Primitive;
25+
use crate::arrays::decimal::DecimalArrayExt;
26+
use crate::arrays::decimal::copy_to_canonical_width;
27+
use crate::dtype::DecimalDType;
28+
use crate::dtype::NativeDecimalType;
29+
use crate::executor::ExecutionCtx;
30+
use crate::executor::ExecutionResult;
31+
use crate::match_each_decimal_value_type;
32+
use crate::require_child;
33+
34+
pub(super) fn execute(
35+
mut array: Array<Interleave>,
36+
ctx: &mut ExecutionCtx,
37+
) -> VortexResult<ExecutionResult> {
38+
let num_values = array.num_values();
39+
array =
40+
require_child!(array, array.array_indices(), InterleaveSlots::ARRAY_INDICES => Primitive);
41+
array = require_child!(array, array.row_indices(), InterleaveSlots::ROW_INDICES => Primitive);
42+
for i in 0..num_values {
43+
array = require_child!(array, array.value(i), InterleaveSlots::VALUES_OFFSET + i => AnyColumnar);
44+
}
45+
46+
let validity = array.as_ref().validity()?;
47+
let dtype = DecimalDType::try_from(array.dtype())?;
48+
let output = match_each_decimal_value_type!(dtype.canonical_values_type(), |T| {
49+
DecimalArray::new(gather_values::<T>(&array, dtype, ctx)?, dtype, validity)
50+
});
51+
52+
Ok(ExecutionResult::done(output))
53+
}
54+
55+
fn gather_values<T: NativeDecimalType>(
56+
array: &Array<Interleave>,
57+
dtype: DecimalDType,
58+
ctx: &mut ExecutionCtx,
59+
) -> VortexResult<Buffer<T>> {
60+
let sources = (0..array.num_values())
61+
.map(|i| prepare_source::<T>(array.value(i), dtype, ctx))
62+
.collect::<VortexResult<Vec<_>>>()?;
63+
64+
gather(array, &sources, ctx.allocator())
65+
}
66+
67+
/// Normalizes one decimal source while retaining a single payload for a constant.
68+
fn prepare_source<T: NativeDecimalType>(
69+
value: &ArrayRef,
70+
dtype: DecimalDType,
71+
ctx: &mut ExecutionCtx,
72+
) -> VortexResult<Source<T>> {
73+
if let Some(constant) = value.as_opt::<Constant>() {
74+
let payload = match constant.scalar().as_decimal().decimal_value() {
75+
Some(value) => value.cast::<T>().ok_or_else(|| {
76+
vortex_err!("interleave decimal constant must fit {dtype}, got {value}")
77+
})?,
78+
None => T::default(),
79+
};
80+
81+
return Ok(Source::constant(payload, value.len(), ctx.allocator()));
82+
}
83+
84+
let decimal = value.as_::<Decimal>();
85+
let data = if decimal.values_type() == T::DECIMAL_TYPE {
86+
decimal.buffer::<T>()
87+
} else {
88+
// TODO(connor): Remove this branch once every `DecimalArray` stores its values at the
89+
// canonical width of its dtype (#10701). Then all sources share the canonical width `T`,
90+
// because they share the dtype up to nullability. Until then, a source can store its values
91+
// at another width. The copy widens such values, or checks their precision and narrows them
92+
// with zeros in the null slots.
93+
let mask = decimal.validity()?.execute_mask(value.len(), ctx)?;
94+
let copy = match_each_decimal_value_type!(decimal.values_type(), |S| {
95+
copy_to_canonical_width(decimal.buffer::<S>().as_slice(), &mask, dtype)?
96+
});
97+
Buffer::<T>::from_byte_buffer(copy.unwrap_host())
98+
};
99+
100+
Ok(Source::decoded(data))
101+
}
102+
103+
#[cfg(test)]
104+
mod tests {
105+
use std::sync::LazyLock;
106+
107+
use rstest::rstest;
108+
use vortex_buffer::Buffer;
109+
use vortex_buffer::buffer;
110+
use vortex_error::VortexExpect;
111+
use vortex_error::VortexResult;
112+
use vortex_session::VortexSession;
113+
114+
use crate::ArrayRef;
115+
use crate::IntoArray;
116+
use crate::VortexSessionExecute;
117+
use crate::arrays::ConstantArray;
118+
use crate::arrays::DecimalArray;
119+
use crate::arrays::InterleaveArray;
120+
use crate::arrays::PrimitiveArray;
121+
use crate::arrays::decimal::DecimalArrayExt;
122+
use crate::assert_arrays_eq;
123+
use crate::dtype::BigCast;
124+
use crate::dtype::DType;
125+
use crate::dtype::DecimalDType;
126+
use crate::dtype::NativeDecimalType;
127+
use crate::dtype::Nullability;
128+
use crate::dtype::i256;
129+
use crate::match_each_decimal_value_type;
130+
use crate::scalar::DecimalValue;
131+
use crate::scalar::Scalar;
132+
use crate::validity::Validity;
133+
134+
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
135+
136+
/// Interleaves `values` with one `(array_index, row_index)` pair for each output row, and
137+
/// checks that the result has the canonical width of its dtype and equals `expected`.
138+
#[track_caller]
139+
fn assert_interleaved(
140+
values: Vec<ArrayRef>,
141+
indices: &[(u8, u8)],
142+
expected: DecimalArray,
143+
) -> VortexResult<()> {
144+
let branches = PrimitiveArray::from_iter(indices.iter().map(|&(branch, _)| branch));
145+
let rows = PrimitiveArray::from_iter(indices.iter().map(|&(_, row)| row));
146+
let array = InterleaveArray::try_new(values, branches.into_array(), rows.into_array())?;
147+
let mut ctx = SESSION.create_execution_ctx();
148+
149+
let output = array.into_array().execute::<DecimalArray>(&mut ctx)?;
150+
assert_eq!(
151+
output.values_type(),
152+
expected.decimal_dtype().canonical_values_type()
153+
);
154+
assert_eq!(output.dtype(), expected.dtype());
155+
assert_arrays_eq!(output.into_array(), expected.into_array(), &mut ctx);
156+
157+
Ok(())
158+
}
159+
160+
/// Gathers rows out of order, with repeats and nulls, at each canonical width. The null slot of
161+
/// the nullable source holds `T::MIN`, which can be outside the precision.
162+
#[rstest]
163+
#[case::i8(2)]
164+
#[case::i16(4)]
165+
#[case::i32(9)]
166+
#[case::i64(18)]
167+
#[case::i128(38)]
168+
#[case::i256(76)]
169+
fn test_decimal_interleave(#[case] precision: u8) -> VortexResult<()> {
170+
let dtype = DecimalDType::new(precision, 2);
171+
match_each_decimal_value_type!(dtype.canonical_values_type(), |T| {
172+
let min = T::MIN_BY_PRECISION[usize::from(precision)];
173+
let max = T::MAX_BY_PRECISION[usize::from(precision)];
174+
let zero = T::default();
175+
let nullable = DecimalArray::new(
176+
Buffer::copy_from([min, T::MIN, max]),
177+
dtype,
178+
Validity::from_iter([true, false, true]),
179+
);
180+
let non_nullable = DecimalArray::new(
181+
Buffer::copy_from([max, min, zero]),
182+
dtype,
183+
Validity::NonNullable,
184+
);
185+
let expected = DecimalArray::new(
186+
Buffer::copy_from([zero, min, zero, max, max, min]),
187+
dtype,
188+
Validity::from_iter([true, true, false, true, true, true]),
189+
);
190+
191+
assert_interleaved(
192+
vec![nullable.into_array(), non_nullable.into_array()],
193+
&[(1, 2), (0, 0), (0, 1), (1, 0), (0, 2), (0, 0)],
194+
expected,
195+
)
196+
})
197+
}
198+
199+
/// A constant keeps one payload at the canonical width, also when its scalar stores the value
200+
/// at another width.
201+
#[rstest]
202+
#[case::narrower_scalar(76, Some(DecimalValue::I8(-7)))]
203+
#[case::wider_scalar(2, Some(DecimalValue::I256(i256::from_i128(-7))))]
204+
#[case::null(9, None)]
205+
fn test_decimal_interleave_constant(
206+
#[case] precision: u8,
207+
#[case] payload: Option<DecimalValue>,
208+
) -> VortexResult<()> {
209+
let dtype = DecimalDType::new(precision, 2);
210+
let (scalar, validity) = match payload {
211+
Some(payload) => (
212+
Scalar::decimal(payload, dtype, Nullability::NonNullable),
213+
Validity::NonNullable,
214+
),
215+
None => (
216+
Scalar::null(DType::Decimal(dtype, Nullability::Nullable)),
217+
Validity::from_iter([false, true, false, false]),
218+
),
219+
};
220+
let constant = ConstantArray::new(scalar, 3).into_array();
221+
match_each_decimal_value_type!(dtype.canonical_values_type(), |T| {
222+
let seven = <T as BigCast>::from(7i8).vortex_expect("7 fits every decimal type");
223+
let minus_seven =
224+
<T as BigCast>::from(-7i8).vortex_expect("-7 fits every decimal type");
225+
let column = DecimalArray::new(buffer![seven], dtype, Validity::NonNullable);
226+
let expected = DecimalArray::new(
227+
buffer![minus_seven, seven, minus_seven, minus_seven],
228+
dtype,
229+
validity,
230+
);
231+
232+
assert_interleaved(
233+
vec![constant, column.into_array()],
234+
&[(0, 2), (1, 0), (0, 1), (0, 2)],
235+
expected,
236+
)
237+
})
238+
}
239+
240+
/// Sources that store their values at another width are copied to the canonical width first.
241+
/// The `i8` source widens, and the `i256` source narrows with garbage in its null slots.
242+
#[test]
243+
fn test_decimal_interleave_mixed_stored_widths() -> VortexResult<()> {
244+
let dtype = DecimalDType::new(9, 0);
245+
let narrow = DecimalArray::new(buffer![1i8, -2], dtype, Validity::NonNullable);
246+
let wide = DecimalArray::new(
247+
buffer![i256::from_i128(3), i256::MIN, i256::MAX],
248+
dtype,
249+
Validity::from_iter([true, false, false]),
250+
);
251+
let expected = DecimalArray::new(
252+
buffer![3i32, -2, 0, 1, 0],
253+
dtype,
254+
Validity::from_iter([true, true, false, true, false]),
255+
);
256+
257+
assert_interleaved(
258+
vec![narrow.into_array(), wide.into_array()],
259+
&[(1, 0), (0, 1), (1, 1), (0, 0), (1, 2)],
260+
expected,
261+
)
262+
}
263+
264+
#[test]
265+
fn test_decimal_interleave_empty() -> VortexResult<()> {
266+
let dtype = DecimalDType::new(9, 0);
267+
let empty = DecimalArray::new(Buffer::<i8>::empty(), dtype, Validity::NonNullable);
268+
let expected = DecimalArray::new(Buffer::<i32>::empty(), dtype, Validity::NonNullable);
269+
270+
assert_interleaved(
271+
vec![empty.clone().into_array(), empty.into_array()],
272+
&[],
273+
expected,
274+
)
275+
}
276+
277+
#[test]
278+
fn test_decimal_interleave_rejects_values_outside_precision() -> VortexResult<()> {
279+
let source = DecimalArray::new(
280+
buffer![100i16],
281+
DecimalDType::new(2, 0),
282+
Validity::NonNullable,
283+
)
284+
.into_array();
285+
let array = InterleaveArray::try_new(
286+
vec![source.clone(), source],
287+
PrimitiveArray::from_iter([0u8]).into_array(),
288+
PrimitiveArray::from_iter([0u8]).into_array(),
289+
)?
290+
.into_array();
291+
292+
let error = array
293+
.execute::<DecimalArray>(&mut SESSION.create_execution_ctx())
294+
.expect_err("the narrowing copy to `i8` rejects 100 at precision 2");
295+
assert!(error.to_string().contains("precision"), "{error}");
296+
297+
Ok(())
298+
}
299+
}

0 commit comments

Comments
 (0)