Skip to content

Commit 1fe8dda

Browse files
authored
Support canonicalising decimal RunEnd arrays (#9665)
Signed-off-by: Onur Satici <[email protected]>
1 parent c9ea0ac commit 1fe8dda

2 files changed

Lines changed: 126 additions & 0 deletions

File tree

‎encodings/runend/src/array.rs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use vortex_array::IntoArray;
2222
use vortex_array::TypedArrayRef;
2323
use vortex_array::VortexSessionExecute;
2424
use vortex_array::array_slots;
25+
use vortex_array::arrays::DecimalArray;
2526
use vortex_array::arrays::Primitive;
2627
use vortex_array::arrays::VarBinViewArray;
2728
use vortex_array::buffer::BufferHandle;
@@ -41,6 +42,7 @@ use vortex_error::vortex_panic;
4142
use vortex_session::VortexSession;
4243
use vortex_session::registry::CachedId;
4344

45+
use crate::compress::runend_decode_decimal;
4446
use crate::compress::runend_decode_primitive;
4547
use crate::compress::runend_decode_varbinview;
4648
use crate::compress::runend_encode;
@@ -486,6 +488,13 @@ pub(super) fn run_end_canonicalize(
486488
let pvalues = array.values().clone().execute_as("values", ctx)?;
487489
runend_decode_primitive(pends, pvalues, array.offset(), array.len(), ctx)?.into_array()
488490
}
491+
DType::Decimal(..) => {
492+
let values = array
493+
.values()
494+
.clone()
495+
.execute_as::<DecimalArray>("values", ctx)?;
496+
runend_decode_decimal(pends, values, array.offset(), array.len(), ctx)?.into_array()
497+
}
489498
DType::Utf8(_) | DType::Binary(_) => {
490499
let values = array
491500
.values()
@@ -503,14 +512,18 @@ mod tests {
503512

504513
use vortex_array::IntoArray;
505514
use vortex_array::VortexSessionExecute;
515+
use vortex_array::arrays::DecimalArray;
506516
use vortex_array::arrays::DictArray;
507517
use vortex_array::arrays::VarBinViewArray;
508518
use vortex_array::assert_arrays_eq;
509519
use vortex_array::builders::VarBinBuilder;
510520
use vortex_array::dtype::DType;
521+
use vortex_array::dtype::DecimalDType;
511522
use vortex_array::dtype::Nullability;
512523
use vortex_array::dtype::PType;
524+
use vortex_array::dtype::i256;
513525
use vortex_buffer::buffer;
526+
use vortex_error::VortexResult;
514527
use vortex_session::VortexSession;
515528

516529
use crate::RunEnd;
@@ -590,4 +603,87 @@ mod tests {
590603
.into_array();
591604
assert_arrays_eq!(arr.into_array(), expected, &mut ctx);
592605
}
606+
607+
#[test]
608+
fn test_runend_decimal_i128() -> VortexResult<()> {
609+
let mut ctx = SESSION.create_execution_ctx();
610+
let decimal_dtype = DecimalDType::new(20, 2);
611+
let values = DecimalArray::from_iter([12_345i128, -67_890, 100], decimal_dtype);
612+
let arr = RunEnd::try_new(
613+
buffer![2u32, 5, 6].into_array(),
614+
values.into_array(),
615+
&mut ctx,
616+
)?;
617+
618+
let decoded = arr.into_array().execute::<DecimalArray>(&mut ctx)?;
619+
let expected = DecimalArray::from_iter(
620+
[12_345i128, 12_345, -67_890, -67_890, -67_890, 100],
621+
decimal_dtype,
622+
);
623+
assert_arrays_eq!(decoded, expected, &mut ctx);
624+
Ok(())
625+
}
626+
627+
#[test]
628+
fn test_runend_decimal_nullable() -> VortexResult<()> {
629+
let mut ctx = SESSION.create_execution_ctx();
630+
let decimal_dtype = DecimalDType::new(20, 2);
631+
let values =
632+
DecimalArray::from_option_iter([Some(12_345i128), None, Some(-67_890)], decimal_dtype);
633+
let arr = RunEnd::try_new(
634+
buffer![2u32, 5, 7].into_array(),
635+
values.into_array(),
636+
&mut ctx,
637+
)?;
638+
639+
let decoded = arr.into_array().execute::<DecimalArray>(&mut ctx)?;
640+
let expected = DecimalArray::from_option_iter(
641+
[
642+
Some(12_345i128),
643+
Some(12_345),
644+
None,
645+
None,
646+
None,
647+
Some(-67_890),
648+
Some(-67_890),
649+
],
650+
decimal_dtype,
651+
);
652+
assert_arrays_eq!(decoded, expected, &mut ctx);
653+
Ok(())
654+
}
655+
656+
#[test]
657+
fn test_runend_decimal_slice() -> VortexResult<()> {
658+
let mut ctx = SESSION.create_execution_ctx();
659+
let decimal_dtype = DecimalDType::new(20, 2);
660+
let values = DecimalArray::from_iter([100i128, 200, 300], decimal_dtype);
661+
let arr = RunEnd::try_new(
662+
buffer![3u32, 5, 10].into_array(),
663+
values.into_array(),
664+
&mut ctx,
665+
)?;
666+
667+
let sliced = arr.slice(2..8)?;
668+
let decoded = sliced.execute::<DecimalArray>(&mut ctx)?;
669+
let expected = DecimalArray::from_iter([100i128, 200, 200, 300, 300, 300], decimal_dtype);
670+
assert_arrays_eq!(decoded, expected, &mut ctx);
671+
Ok(())
672+
}
673+
674+
#[test]
675+
fn test_runend_decimal_i256() -> VortexResult<()> {
676+
let mut ctx = SESSION.create_execution_ctx();
677+
let decimal_dtype = DecimalDType::new(40, 4);
678+
let first = i256::from_i128(123_456);
679+
let second = i256::from_i128(-789_012);
680+
let values = DecimalArray::from_iter([first, second], decimal_dtype);
681+
let arr = RunEnd::try_new(buffer![2u32, 5].into_array(), values.into_array(), &mut ctx)?;
682+
683+
let decoded = arr.into_array().execute::<DecimalArray>(&mut ctx)?;
684+
let expected =
685+
DecimalArray::from_iter([first, first, second, second, second], decimal_dtype);
686+
assert_arrays_eq!(decoded, expected, &mut ctx);
687+
Ok(())
688+
}
593689
}

‎encodings/runend/src/compress.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,19 @@ use vortex_array::ExecutionCtx;
88
use vortex_array::IntoArray;
99
use vortex_array::arrays::BoolArray;
1010
use vortex_array::arrays::ConstantArray;
11+
use vortex_array::arrays::DecimalArray;
1112
use vortex_array::arrays::Primitive;
1213
use vortex_array::arrays::PrimitiveArray;
1314
use vortex_array::arrays::VarBinViewArray;
1415
use vortex_array::arrays::bool::BoolArrayExt;
16+
use vortex_array::arrays::decimal::DecimalArrayExt;
1517
use vortex_array::arrays::primitive::PrimitiveArrayExt;
1618
use vortex_array::buffer::BufferHandle;
1719
use vortex_array::dtype::NativePType;
1820
use vortex_array::dtype::Nullability;
1921
use vortex_array::expr::stats::Precision;
2022
use vortex_array::expr::stats::Stat;
23+
use vortex_array::match_each_decimal_value_type;
2124
use vortex_array::match_each_native_ptype;
2225
use vortex_array::match_each_unsigned_integer_ptype;
2326
use vortex_array::scalar::Scalar;
@@ -284,6 +287,33 @@ pub fn runend_decode_typed_primitive<T: NativePType>(
284287
PrimitiveArray::new(decoded, validity)
285288
}
286289

290+
/// Decode a run-end encoded decimal array by expanding its native values.
291+
pub fn runend_decode_decimal(
292+
ends: PrimitiveArray,
293+
values: DecimalArray,
294+
offset: usize,
295+
length: usize,
296+
ctx: &mut ExecutionCtx,
297+
) -> VortexResult<DecimalArray> {
298+
let validity_mask = values
299+
.as_ref()
300+
.validity()?
301+
.execute_mask(values.as_ref().len(), ctx)?;
302+
303+
Ok(match_each_decimal_value_type!(values.values_type(), |D| {
304+
let (decoded, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| {
305+
runend_decode_slice(
306+
trimmed_ends_iter(ends.as_slice::<E>(), offset, length),
307+
values.buffer::<D>().as_slice(),
308+
validity_mask,
309+
values.dtype().nullability(),
310+
length,
311+
)
312+
});
313+
DecimalArray::new(decoded, values.decimal_dtype(), validity)
314+
}))
315+
}
316+
287317
/// Decode a run-end encoded VarBinView array by expanding views directly.
288318
pub fn runend_decode_varbinview(
289319
ends: PrimitiveArray,

0 commit comments

Comments
 (0)