Skip to content

Commit 1dffc55

Browse files
authored
Remove vortex-compute (vortex-data#6561)
## Summary Tracking Issue: vortex-data#6547 Removes `vortex-compute` and migrates some of the code that was used / should have been used from `vortex-compute` in `vortex-array` inline. This is part of the effort to merge `vortex-dtype` into `vortex-array`. `vortex-compute` was just in the way because it depends on the `vortex-dtype` crate but `vortex-array` depends on `vortex-compute`. ## API Changes Removes all of `vortex-compute`. ## Testing N/A --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent 99ba753 commit 1dffc55

46 files changed

Lines changed: 1022 additions & 4871 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,7 @@ jobs:
667667
include:
668668
- { shard: 1, name: "Core foundation", packages: "vortex-buffer vortex-error" }
669669
- { shard: 2, name: "Arrays", packages: "vortex-array vortex-dtype", features: "--features _test-harness" }
670-
- { shard: 3, name: "Main library", packages: "vortex vortex-compute" }
670+
- { shard: 3, name: "Main library", packages: "vortex" }
671671
- { shard: 4, name: "Encodings 1", packages: "vortex-alp vortex-bytebool vortex-datetime-parts" }
672672
- { shard: 5, name: "Encodings 2", packages: "vortex-decimal-byte-parts vortex-fastlanes vortex-fsst", features: "--features _test-harness" }
673673
- { shard: 6, name: "Encodings 3", packages: "vortex-pco vortex-runend vortex-sequence" }

‎Cargo.lock‎

Lines changed: 0 additions & 24 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ members = [
1111
"vortex-io",
1212
"vortex-proto",
1313
"vortex-dtype",
14-
"vortex-compute",
1514
"vortex-array",
1615
"vortex-btrblocks",
1716
"vortex-layout",
@@ -253,7 +252,6 @@ vortex-array = { version = "0.1.0", path = "./vortex-array", default-features =
253252
vortex-btrblocks = { version = "0.1.0", path = "./vortex-btrblocks", default-features = false }
254253
vortex-buffer = { version = "0.1.0", path = "./vortex-buffer", default-features = false }
255254
vortex-bytebool = { version = "0.1.0", path = "./encodings/bytebool", default-features = false }
256-
vortex-compute = { version = "0.1.0", path = "./vortex-compute", default-features = false }
257255
vortex-datafusion = { version = "0.1.0", path = "./vortex-datafusion", default-features = false }
258256
vortex-datetime-parts = { version = "0.1.0", path = "./encodings/datetime-parts", default-features = false }
259257
vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-parts", default-features = false }

‎encodings/fastlanes/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ prost = { workspace = true }
2626
rand = { workspace = true, optional = true }
2727
vortex-array = { workspace = true }
2828
vortex-buffer = { workspace = true }
29-
vortex-compute = { workspace = true }
3029
vortex-dtype = { workspace = true }
3130
vortex-error = { workspace = true }
3231
vortex-mask = { workspace = true }

‎vortex-array/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ tabled = { workspace = true, optional = true, default-features = false, features
6262
termtree = { workspace = true }
6363
tracing = { workspace = true }
6464
vortex-buffer = { workspace = true, features = ["arrow"] }
65-
vortex-compute = { workspace = true, default-features = true }
6665
vortex-dtype = { workspace = true, features = ["serde"] }
6766
vortex-error = { workspace = true }
6867
vortex-flatbuffers = { workspace = true, features = ["array"] }
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! [`BitBuffer`] filtering algorithms.
5+
6+
use vortex_buffer::BitBuffer;
7+
use vortex_buffer::BitBufferMut;
8+
use vortex_buffer::get_bit;
9+
use vortex_mask::MaskValues;
10+
11+
/// Filter a [`BitBuffer`] by [`MaskValues`], returning a new [`BitBuffer`].
12+
pub(super) fn filter_bit_buffer(bb: &BitBuffer, mask: &MaskValues) -> BitBuffer {
13+
assert_eq!(
14+
mask.len(),
15+
bb.len(),
16+
"Selection mask length must equal the mask length"
17+
);
18+
19+
// BitBuffer filtering always uses indices for simplicity.
20+
filter_bitbuffer_by_indices(bb, mask.indices())
21+
}
22+
23+
fn filter_bitbuffer_by_indices(bb: &BitBuffer, indices: &[usize]) -> BitBuffer {
24+
let bools = bb.inner().as_ref();
25+
let bit_offset = bb.offset();
26+
27+
// FIXME(ngates): this is slower than it could be!
28+
BitBufferMut::collect_bool(indices.len(), |idx| {
29+
let idx = *unsafe { indices.get_unchecked(idx) };
30+
get_bit(bools, bit_offset + idx) // Panics if out of bounds.
31+
})
32+
.freeze()
33+
}
34+
35+
#[allow(unused)]
36+
fn filter_bitbuffer_by_slices(bb: &BitBuffer, slices: &[(usize, usize)]) -> BitBuffer {
37+
let bools = bb.inner().as_ref();
38+
let bit_offset = bb.offset();
39+
let output_len: usize = slices.iter().map(|(start, end)| end - start).sum();
40+
41+
let mut out = BitBufferMut::with_capacity(output_len);
42+
43+
// FIXME(ngates): this is slower than it could be!
44+
for &(start, end) in slices {
45+
for idx in start..end {
46+
out.append(get_bit(bools, bit_offset + idx)); // Panics if out of bounds.
47+
}
48+
}
49+
50+
out.freeze()
51+
}
52+
53+
#[cfg(test)]
54+
mod tests {
55+
use vortex_buffer::bitbuffer;
56+
57+
use crate::arrays::filter::execute::bitbuffer::filter_bitbuffer_by_indices;
58+
59+
#[test]
60+
fn filter_bool_by_index_test() {
61+
let buf = bitbuffer![1 1 0];
62+
let indices = [0usize, 2];
63+
let filtered = filter_bitbuffer_by_indices(&buf, &indices);
64+
assert_eq!(2, filtered.len());
65+
assert_eq!(filtered, bitbuffer![1 0])
66+
}
67+
}

‎vortex-array/src/arrays/filter/execute/bool.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,19 @@
33

44
use std::sync::Arc;
55

6-
use vortex_compute::filter::Filter;
76
use vortex_error::VortexExpect;
87
use vortex_mask::MaskValues;
98

109
use crate::arrays::BoolArray;
10+
use crate::arrays::filter::execute::bitbuffer;
1111
use crate::arrays::filter::execute::filter_validity;
1212

1313
pub fn filter_bool(array: &BoolArray, mask: &Arc<MaskValues>) -> BoolArray {
1414
let validity = array.validity().vortex_expect("missing BoolArray validity");
1515
let filtered_validity = filter_validity(validity, mask);
1616

1717
let bit_buffer = array.to_bit_buffer();
18-
let filtered_buffer = bit_buffer.filter(mask.as_ref());
18+
let filtered_buffer = bitbuffer::filter_bit_buffer(&bit_buffer, mask.as_ref());
1919

2020
BoolArray::new(filtered_buffer, filtered_validity)
2121
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Buffer-level filter dispatch.
5+
//!
6+
//! Provides [`filter_buffer`] which filters a [`Buffer<T>`] by [`MaskValues`], attempting an
7+
//! in-place filter when the buffer has exclusive ownership.
8+
9+
use vortex_buffer::Buffer;
10+
use vortex_mask::MaskValues;
11+
12+
use crate::arrays::filter::execute::slice;
13+
14+
/// Filter a [`Buffer<T>`] by [`MaskValues`], returning a new buffer.
15+
///
16+
/// This will attempt to filter in-place (via [`Buffer::try_into_mut`]) when the buffer has
17+
/// exclusive ownership, avoiding an extra allocation.
18+
pub(super) fn filter_buffer<T: Copy>(buffer: Buffer<T>, mask: &MaskValues) -> Buffer<T> {
19+
match buffer.try_into_mut() {
20+
Ok(mut buffer_mut) => {
21+
let new_len = slice::filter_slice_mut_by_mask_values(buffer_mut.as_mut_slice(), mask);
22+
buffer_mut.truncate(new_len);
23+
buffer_mut.freeze()
24+
}
25+
// Otherwise, allocate a new buffer and fill it in.
26+
Err(buffer) => slice::filter_slice_by_mask_values(buffer.as_slice(), mask),
27+
}
28+
}
29+
30+
#[cfg(test)]
31+
mod tests {
32+
use vortex_buffer::BufferMut;
33+
use vortex_buffer::buffer;
34+
use vortex_mask::Mask;
35+
36+
use super::*;
37+
38+
// Helper to get `MaskValues` from a `Mask`.
39+
fn mask_values(mask: &Mask) -> &MaskValues {
40+
match mask {
41+
Mask::Values(v) => v.as_ref(),
42+
_ => panic!("expected Mask::Values"),
43+
}
44+
}
45+
46+
#[test]
47+
fn test_filter_buffer_by_indices() {
48+
let buf = buffer![10u32, 20, 30, 40, 50];
49+
let mask = Mask::from_iter([true, false, true, false, true]);
50+
51+
let result = filter_buffer(buf, mask_values(&mask));
52+
assert_eq!(result, buffer![10u32, 30, 50]);
53+
}
54+
55+
#[test]
56+
fn test_filter_indices_direct() {
57+
let buf = buffer![100u32, 200, 300, 400];
58+
let mask = Mask::from_iter([true, false, true, true]);
59+
let result = filter_buffer(buf, mask_values(&mask));
60+
assert_eq!(result, buffer![100u32, 300, 400]);
61+
}
62+
63+
#[test]
64+
fn test_filter_sparse() {
65+
let buf = Buffer::from(BufferMut::from_iter(0u32..1000));
66+
// Keep every third element.
67+
let mask = Mask::from_iter((0..1000).map(|i| i % 3 == 0));
68+
69+
let result = filter_buffer(buf, mask_values(&mask));
70+
let expected: Vec<u32> = (0..1000).filter(|i| i % 3 == 0).collect();
71+
assert_eq!(result.as_slice(), &expected[..]);
72+
}
73+
74+
#[test]
75+
fn test_filter_dense() {
76+
let buf = buffer![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10];
77+
// Dense selection (80% selected).
78+
let mask = Mask::from_iter([true, true, true, true, false, true, true, true, false, true]);
79+
80+
let result = filter_buffer(buf, mask_values(&mask));
81+
assert_eq!(result, buffer![1u32, 2, 3, 4, 6, 7, 8, 10]);
82+
}
83+
}

‎vortex-array/src/arrays/filter/execute/decimal.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,19 @@
33

44
use std::sync::Arc;
55

6-
use vortex_compute::filter::Filter;
76
use vortex_dtype::match_each_decimal_value_type;
87
use vortex_mask::MaskValues;
98

109
use crate::arrays::DecimalArray;
10+
use crate::arrays::filter::execute::buffer;
1111
use crate::arrays::filter::execute::filter_validity;
1212
use crate::vtable::ValidityHelper;
1313

1414
pub fn filter_decimal(array: &DecimalArray, mask: &Arc<MaskValues>) -> DecimalArray {
1515
let filtered_validity = filter_validity(array.validity().clone(), mask);
1616

1717
match_each_decimal_value_type!(array.values_type(), |T| {
18-
let filtered_buffer = array.buffer::<T>().filter(mask.as_ref());
18+
let filtered_buffer = buffer::filter_buffer(array.buffer::<T>(), mask.as_ref());
1919

2020
// SAFETY: We filter both the validity and the buffer with the same mask, so they must have
2121
// the same length, and since the buffer came from an existing and valid `DecimalArray` the

‎vortex-array/src/arrays/filter/execute/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,14 @@ use crate::arrays::NullArray;
2323
use crate::scalar::Scalar;
2424
use crate::validity::Validity;
2525

26+
mod bitbuffer;
2627
mod bool;
28+
mod buffer;
2729
mod decimal;
2830
mod fixed_size_list;
2931
mod listview;
3032
mod primitive;
33+
mod slice;
3134
mod struct_;
3235
mod varbinview;
3336

0 commit comments

Comments
 (0)