Skip to content

Commit a38a6cc

Browse files
Document bitmap filter type support
1 parent b717134 commit a38a6cc

1 file changed

Lines changed: 23 additions & 3 deletions

File tree

‎datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,19 @@ use std::hash::{Hash, Hasher};
2929
use super::result::build_in_list_result;
3030
use super::static_filter::{StaticFilter, handle_dictionary};
3131

32+
/// Storage for the bits used by [`BitmapFilter`].
33+
///
34+
/// `BitmapFilter` represents an `IN` list with one bit for each possible
35+
/// value, so membership checks become direct bit tests. This trait lets the
36+
/// same filter code use different storage sizes for different integer widths.
3237
pub(super) trait BitmapStorage: Send + Sync {
3338
fn new_zeroed() -> Self;
3439
fn set_bit(&mut self, index: usize);
3540
fn get_bit(&self, index: usize) -> bool;
3641
}
3742

43+
// `UInt8` has 256 possible values, 0 through 255. One bit per value takes
44+
// 256 bits, which fits in four `u64` words.
3845
impl BitmapStorage for [u64; 4] {
3946
#[inline]
4047
fn new_zeroed() -> Self {
@@ -50,6 +57,9 @@ impl BitmapStorage for [u64; 4] {
5057
}
5158
}
5259

60+
// `UInt16` has 65,536 possible values. One bit per value takes 65,536 bits,
61+
// which is 1,024 `u64` words, or 8 KiB. Box the array so the filter stores a
62+
// pointer instead of carrying an 8 KiB array inline.
5363
impl BitmapStorage for Box<[u64; 1024]> {
5464
#[inline]
5565
fn new_zeroed() -> Self {
@@ -65,24 +75,34 @@ impl BitmapStorage for Box<[u64; 1024]> {
6575
}
6676
}
6777

78+
/// Arrow primitive types supported by [`BitmapFilter`].
79+
///
80+
/// Arrow already defines the Rust value type as `T::Native`. This trait only
81+
/// supplies the bitmap storage size for the two integer domains that are small
82+
/// enough to represent with one bit per possible value.
6883
pub(super) trait BitmapFilterType:
6984
ArrowPrimitiveType + Send + Sync + 'static
7085
{
7186
type Storage: BitmapStorage;
7287
}
7388

89+
/// `UInt8` has 256 possible values, so four `u64` words cover the full domain.
7490
impl BitmapFilterType for UInt8Type {
7591
type Storage = [u64; 4];
7692
}
7793

94+
/// `UInt16` has 65,536 possible values, so 1,024 `u64` words cover the full
95+
/// domain.
7896
impl BitmapFilterType for UInt16Type {
7997
type Storage = Box<[u64; 1024]>;
8098
}
8199

82-
/// Bitmap filter for O(1) set membership via single bit test.
100+
/// `IN` filter backed by one bit per possible value.
83101
///
84-
/// Small integer domains can store membership in a fixed-size bitmap instead
85-
/// of using a hash table.
102+
/// Building the filter scans the non-null values in the IN-list and turns on
103+
/// the bit selected by each value. Evaluating input values checks the same bit
104+
/// position. Null handling and `NOT IN` inversion are handled by
105+
/// `build_in_list_result`.
86106
pub(super) struct BitmapFilter<T: BitmapFilterType> {
87107
null_count: usize,
88108
bits: T::Storage,

0 commit comments

Comments
 (0)