Skip to content

Commit 73b4b7b

Browse files
committed
Implement compare in vortex instead of falling back to arrow
Signed-off-by: Robert Kruszewski <[email protected]>
1 parent b9779d6 commit 73b4b7b

12 files changed

Lines changed: 2381 additions & 624 deletions

File tree

‎vortex-array/benches/compare.rs‎

Lines changed: 178 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,21 @@ use rand::RngExt;
88
use rand::SeedableRng;
99
use rand::distr::Uniform;
1010
use rand::prelude::StdRng;
11+
use vortex_array::ArrayRef;
1112
use vortex_array::Canonical;
1213
use vortex_array::IntoArray;
1314
use vortex_array::VortexSessionExecute;
1415
use vortex_array::arrays::BoolArray;
16+
use vortex_array::arrays::ConstantArray;
17+
use vortex_array::arrays::DecimalArray;
18+
use vortex_array::arrays::PrimitiveArray;
19+
use vortex_array::arrays::StructArray;
20+
use vortex_array::arrays::VarBinViewArray;
1521
use vortex_array::builtins::ArrayBuiltins;
22+
use vortex_array::dtype::DecimalDType;
23+
use vortex_array::scalar::Scalar;
1624
use vortex_array::scalar_fn::fns::operators::Operator;
25+
use vortex_array::validity::Validity;
1726
use vortex_buffer::Buffer;
1827

1928
fn main() {
@@ -22,51 +31,190 @@ fn main() {
2231

2332
const ARRAY_SIZE: usize = 65_536;
2433

25-
#[divan::bench]
26-
fn compare_bool(bencher: Bencher) {
27-
let mut rng = StdRng::seed_from_u64(0);
28-
let range = Uniform::new(0u8, 1).unwrap();
29-
30-
let arr1 = BoolArray::from_iter((0..ARRAY_SIZE).map(|_| rng.sample(range) == 0)).into_array();
31-
let arr2 = BoolArray::from_iter((0..ARRAY_SIZE).map(|_| rng.sample(range) == 0)).into_array();
34+
fn bench_compare(bencher: Bencher, lhs: ArrayRef, rhs: ArrayRef, op: Operator) {
3235
let session = vortex_array::array_session();
33-
3436
bencher
35-
.with_inputs(|| (&arr1, &arr2, session.create_execution_ctx()))
37+
.with_inputs(|| (&lhs, &rhs, session.create_execution_ctx()))
3638
.bench_refs(|input| {
3739
input
3840
.0
3941
.clone()
40-
.binary(input.1.clone(), Operator::Gte)
42+
.binary(input.1.clone(), op)
4143
.unwrap()
4244
.execute::<Canonical>(&mut input.2)
4345
});
4446
}
4547

46-
#[divan::bench]
47-
fn compare_int(bencher: Bencher) {
48-
let mut rng = StdRng::seed_from_u64(0);
49-
let range = Uniform::new(0i64, 100_000_000).unwrap();
48+
fn bool_array(rng: &mut StdRng) -> ArrayRef {
49+
BoolArray::from_iter((0..ARRAY_SIZE).map(|_| rng.random_bool(0.5))).into_array()
50+
}
51+
52+
fn bool_array_nullable(rng: &mut StdRng) -> ArrayRef {
53+
BoolArray::new(
54+
(0..ARRAY_SIZE).map(|_| rng.random_bool(0.5)).collect(),
55+
Validity::from_iter((0..ARRAY_SIZE).map(|_| rng.random_bool(0.9))),
56+
)
57+
.into_array()
58+
}
5059

51-
let arr1 = (0..ARRAY_SIZE)
60+
fn int_array(rng: &mut StdRng) -> ArrayRef {
61+
let range = Uniform::new(0i64, 100_000_000).unwrap();
62+
(0..ARRAY_SIZE)
5263
.map(|_| rng.sample(range))
5364
.collect::<Buffer<_>>()
54-
.into_array();
65+
.into_array()
66+
}
5567

56-
let arr2 = (0..ARRAY_SIZE)
57-
.map(|_| rng.sample(range))
68+
fn int_array_nullable(rng: &mut StdRng) -> ArrayRef {
69+
let range = Uniform::new(0i64, 100_000_000).unwrap();
70+
PrimitiveArray::new(
71+
(0..ARRAY_SIZE)
72+
.map(|_| rng.sample(range))
73+
.collect::<Buffer<_>>(),
74+
Validity::from_iter((0..ARRAY_SIZE).map(|_| rng.random_bool(0.9))),
75+
)
76+
.into_array()
77+
}
78+
79+
fn float_array(rng: &mut StdRng) -> ArrayRef {
80+
(0..ARRAY_SIZE)
81+
.map(|_| rng.random_range(0.0f64..1.0))
5882
.collect::<Buffer<_>>()
59-
.into_array();
60-
let session = vortex_array::array_session();
83+
.into_array()
84+
}
6185

62-
bencher
63-
.with_inputs(|| (&arr1, &arr2, session.create_execution_ctx()))
64-
.bench_refs(|input| {
65-
input
66-
.0
67-
.clone()
68-
.binary(input.1.clone(), Operator::Gte)
69-
.unwrap()
70-
.execute::<Canonical>(&mut input.2)
71-
});
86+
fn string_array(rng: &mut StdRng) -> ArrayRef {
87+
VarBinViewArray::from_iter_str((0..ARRAY_SIZE).map(|_| {
88+
let len = rng.random_range(1usize..24);
89+
(0..len)
90+
.map(|_| char::from(rng.random_range(b'a'..=b'z')))
91+
.collect::<String>()
92+
}))
93+
.into_array()
94+
}
95+
96+
fn decimal_array(rng: &mut StdRng) -> ArrayRef {
97+
DecimalArray::from_iter::<i128, _>(
98+
(0..ARRAY_SIZE).map(|_| rng.random_range(0i128..100_000_000)),
99+
DecimalDType::new(38, 2),
100+
)
101+
.into_array()
102+
}
103+
104+
#[divan::bench]
105+
fn compare_bool(bencher: Bencher) {
106+
let mut rng = StdRng::seed_from_u64(0);
107+
let arr1 = bool_array(&mut rng);
108+
let arr2 = bool_array(&mut rng);
109+
bench_compare(bencher, arr1, arr2, Operator::Gte);
110+
}
111+
112+
#[divan::bench]
113+
fn compare_bool_nullable(bencher: Bencher) {
114+
let mut rng = StdRng::seed_from_u64(0);
115+
let arr1 = bool_array_nullable(&mut rng);
116+
let arr2 = bool_array_nullable(&mut rng);
117+
bench_compare(bencher, arr1, arr2, Operator::Gte);
118+
}
119+
120+
#[divan::bench]
121+
fn compare_bool_constant(bencher: Bencher) {
122+
let mut rng = StdRng::seed_from_u64(0);
123+
let arr = bool_array(&mut rng);
124+
let constant = ConstantArray::new(true, ARRAY_SIZE).into_array();
125+
bench_compare(bencher, arr, constant, Operator::Eq);
126+
}
127+
128+
#[divan::bench]
129+
fn compare_int(bencher: Bencher) {
130+
let mut rng = StdRng::seed_from_u64(0);
131+
let arr1 = int_array(&mut rng);
132+
let arr2 = int_array(&mut rng);
133+
bench_compare(bencher, arr1, arr2, Operator::Gte);
134+
}
135+
136+
#[divan::bench]
137+
fn compare_int_nullable(bencher: Bencher) {
138+
let mut rng = StdRng::seed_from_u64(0);
139+
let arr1 = int_array_nullable(&mut rng);
140+
let arr2 = int_array_nullable(&mut rng);
141+
bench_compare(bencher, arr1, arr2, Operator::Gte);
142+
}
143+
144+
#[divan::bench]
145+
fn compare_int_constant(bencher: Bencher) {
146+
let mut rng = StdRng::seed_from_u64(0);
147+
let arr = int_array(&mut rng);
148+
let constant = ConstantArray::new(50_000_000i64, ARRAY_SIZE).into_array();
149+
bench_compare(bencher, arr, constant, Operator::Gte);
150+
}
151+
152+
#[divan::bench]
153+
fn compare_int_eq(bencher: Bencher) {
154+
let mut rng = StdRng::seed_from_u64(0);
155+
let arr1 = int_array(&mut rng);
156+
let arr2 = int_array(&mut rng);
157+
bench_compare(bencher, arr1, arr2, Operator::Eq);
158+
}
159+
160+
#[divan::bench]
161+
fn compare_float(bencher: Bencher) {
162+
let mut rng = StdRng::seed_from_u64(0);
163+
let arr1 = float_array(&mut rng);
164+
let arr2 = float_array(&mut rng);
165+
bench_compare(bencher, arr1, arr2, Operator::Gte);
166+
}
167+
168+
#[divan::bench]
169+
fn compare_decimal(bencher: Bencher) {
170+
let mut rng = StdRng::seed_from_u64(0);
171+
let arr1 = decimal_array(&mut rng);
172+
let arr2 = decimal_array(&mut rng);
173+
bench_compare(bencher, arr1, arr2, Operator::Gte);
174+
}
175+
176+
#[divan::bench]
177+
fn compare_string_eq(bencher: Bencher) {
178+
let mut rng = StdRng::seed_from_u64(0);
179+
let arr1 = string_array(&mut rng);
180+
let arr2 = string_array(&mut rng);
181+
bench_compare(bencher, arr1, arr2, Operator::Eq);
182+
}
183+
184+
#[divan::bench]
185+
fn compare_string_lt(bencher: Bencher) {
186+
let mut rng = StdRng::seed_from_u64(0);
187+
let arr1 = string_array(&mut rng);
188+
let arr2 = string_array(&mut rng);
189+
bench_compare(bencher, arr1, arr2, Operator::Lt);
190+
}
191+
192+
#[divan::bench]
193+
fn compare_string_constant(bencher: Bencher) {
194+
let mut rng = StdRng::seed_from_u64(0);
195+
let arr = string_array(&mut rng);
196+
let constant = ConstantArray::new(Scalar::from("mmmmmmmmmmmm"), ARRAY_SIZE).into_array();
197+
bench_compare(bencher, arr, constant, Operator::Lt);
198+
}
199+
200+
fn struct_array(rng: &mut StdRng) -> ArrayRef {
201+
StructArray::from_fields(&[("a", int_array(rng)), ("b", int_array(rng))])
202+
.unwrap()
203+
.into_array()
204+
}
205+
206+
#[divan::bench]
207+
fn compare_struct_lt(bencher: Bencher) {
208+
let mut rng = StdRng::seed_from_u64(0);
209+
let arr1 = struct_array(&mut rng);
210+
let arr2 = struct_array(&mut rng);
211+
bench_compare(bencher, arr1, arr2, Operator::Lt);
212+
}
213+
214+
#[divan::bench]
215+
fn compare_struct_eq(bencher: Bencher) {
216+
let mut rng = StdRng::seed_from_u64(0);
217+
let arr1 = struct_array(&mut rng);
218+
let arr2 = struct_array(&mut rng);
219+
bench_compare(bencher, arr1, arr2, Operator::Eq);
72220
}

‎vortex-array/src/arrays/extension/compute/compare.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ impl CompareKernel for Extension {
2222
operator: CompareOperator,
2323
_ctx: &mut ExecutionCtx,
2424
) -> VortexResult<Option<ArrayRef>> {
25+
// Storage values are only comparable when both sides share the same extension dtype
26+
// (e.g. timestamps in different units must not compare their raw storage).
27+
if !lhs.dtype().eq_ignore_nullability(rhs.dtype()) {
28+
return Ok(None);
29+
}
30+
2531
// If the RHS is a constant, we can extract the storage scalar.
2632
if let Some(const_ext) = rhs.as_constant() {
2733
let storage_scalar = const_ext.as_extension().to_storage_scalar();

0 commit comments

Comments
 (0)