Skip to content

Commit b161445

Browse files
authored
FoRScheme: declare and refine its bit-packed child (#10757)
## Summary `FoRScheme` bit-packed its child by calling `BITPACKING_V1` directly, without declaring the bit-packing IDs. The builder kept FoR when `fastlanes.bitpacked` (or `vortex.patched`) was disallowed, so the write failed, and the child never used `fastlanes.bitpacked.v2`. FoR now holds the `BitPackingScheme` it uses, declares that scheme's IDs, and refines it alongside the references. Closes #10751. ## Changes - `FoRScheme` gets a `bitpacking` field, and `produced_encodings` appends its IDs. - `refine` picks the references and the child's bit widths independently (four statics). - The child is compressed with FoR's context rather than a leaf one, so v2 block offsets can cascade. Core-edition and CUDA output is unchanged. Sessions that allow `fastlanes.bitpacked.v2` now get blocked children under FoR (fused decode from #10251). ## Testing - `builder.rs`: FoR is dropped without `fastlanes.bitpacked` and kept with it. - `for_.rs`, `tests/for_config.rs`: FoR refines to a blocked child when `fastlanes.bitpacked.v2` is allowed, and the compressed output serializes it. ## Follow-up Excluding `BitPackingScheme` by scheme ID still doesn't stop FoR from bit-packing (documented on `FoRScheme`). --------- Signed-off-by: Matt Katz <[email protected]>
1 parent 066acfb commit b161445

4 files changed

Lines changed: 87 additions & 18 deletions

File tree

‎vortex-btrblocks/src/builder.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ mod tests {
258258
use vortex_decimal_byte_parts::decimal_byte_parts_v1_id;
259259
use vortex_fastlanes::bitpacked_v1_id;
260260
use vortex_fastlanes::bitpacked_v2_id;
261+
use vortex_fastlanes::for_v1_id;
261262
use vortex_fsst::FSST;
262263

263264
use super::*;
@@ -288,6 +289,12 @@ mod tests {
288289
vec![bitpacked_v2_id(), Patched.id()],
289290
false
290291
)]
292+
#[case::for_without_bitpacked(&integer::FOR_V1, vec![for_v1_id()], false)]
293+
#[case::for_with_bitpacked(
294+
&integer::FOR_V1,
295+
vec![for_v1_id(), bitpacked_v1_id(), Patched.id()],
296+
true
297+
)]
291298
fn test_allowed_schemes(
292299
#[case] scheme: &'static dyn Scheme,
293300
#[case] allowed: Vec<ArrayId>,

‎vortex-btrblocks/src/schemes/integer/for_.rs‎

Lines changed: 54 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,11 @@ use vortex_fastlanes::FoR;
2323
use vortex_fastlanes::FoRArrayExt;
2424
use vortex_fastlanes::FoRArraySlotsExt;
2525
use vortex_fastlanes::FoRReferences;
26+
use vortex_fastlanes::bitpacked_v2_id;
2627
use vortex_fastlanes::for_v1_id;
2728
use vortex_fastlanes::for_v2_id;
2829

29-
use super::BITPACKING_V1;
30+
use super::BitPackingScheme;
3031
use crate::ArrayAndStats;
3132
use crate::CascadingCompressor;
3233
use crate::CompressorContext;
@@ -45,32 +46,51 @@ pub(crate) static FOR_V1: FoRScheme = FoRScheme::v1();
4546
/// The FoR scheme that may produce one reference per 1024-element chunk.
4647
pub(crate) static FOR_V2: FoRScheme = FoRScheme::v2();
4748

49+
/// [`FOR_V1`] with a child that may give each 1024-element block its own bit width.
50+
static FOR_V1_BLOCKED: FoRScheme = FoRScheme {
51+
mode: FoRSchemeMode::V1,
52+
bitpacking: BitPackingScheme::v2(),
53+
};
54+
55+
/// [`FOR_V2`] with a child that may give each 1024-element block its own bit width.
56+
static FOR_V2_BLOCKED: FoRScheme = FoRScheme {
57+
mode: FoRSchemeMode::V2,
58+
bitpacking: BitPackingScheme::v2(),
59+
};
60+
4861
/// Frame of Reference encoding.
4962
///
5063
/// The v1 mode subtracts one reference, the array's minimum. The v2 mode subtracts one reference
5164
/// per 1024-element chunk, the chunk's minimum. Arrays with a single reference, including v2
5265
/// arrays whose references compress to a constant, serialize as `fastlanes.for`, while per-chunk
5366
/// references serialize as `fastlanes.for.v2`.
5467
///
55-
/// The default uses v1. [`refine`](Scheme::refine) picks v2 when the v2 ID is allowed and v1
56-
/// otherwise.
68+
/// The encoded child is always bit-packed by a [`BitPackingScheme`] called directly, not chosen by
69+
/// the compressor, so this scheme declares the child's serialized IDs as its own. Excluding the
70+
/// BitPacking scheme does not stop FoR from bit-packing; disallowing `fastlanes.bitpacked` does.
71+
///
72+
/// The default uses v1 for both. [`refine`](Scheme::refine) picks v2 references when
73+
/// `fastlanes.for.v2` is allowed, and a v2 child when `fastlanes.bitpacked.v2` is allowed.
5774
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
5875
pub struct FoRScheme {
5976
mode: FoRSchemeMode,
77+
bitpacking: BitPackingScheme,
6078
}
6179

6280
impl FoRScheme {
6381
/// Creates a FoR scheme configured for v1, which uses a single reference.
6482
pub const fn v1() -> Self {
6583
Self {
6684
mode: FoRSchemeMode::V1,
85+
bitpacking: BitPackingScheme::v1(),
6786
}
6887
}
6988

7089
/// Creates a FoR scheme configured for v2, which may use one reference per chunk.
7190
pub const fn v2() -> Self {
7291
Self {
7392
mode: FoRSchemeMode::V2,
93+
bitpacking: BitPackingScheme::v1(),
7494
}
7595
}
7696
}
@@ -91,17 +111,20 @@ impl Scheme for FoRScheme {
91111
}
92112

93113
fn produced_encodings(&self) -> Vec<ArrayId> {
94-
match self.mode {
114+
let mut encodings = match self.mode {
95115
FoRSchemeMode::V1 => vec![for_v1_id()],
96116
FoRSchemeMode::V2 => vec![for_v1_id(), for_v2_id()],
97-
}
117+
};
118+
encodings.extend(self.bitpacking.produced_encodings());
119+
encodings
98120
}
99121

100122
fn refine(&self, allowed: &dyn Fn(&ArrayId) -> bool) -> &dyn Scheme {
101-
if allowed(&for_v2_id()) {
102-
&FOR_V2
103-
} else {
104-
&FOR_V1
123+
match (allowed(&for_v2_id()), allowed(&bitpacked_v2_id())) {
124+
(false, false) => &FOR_V1,
125+
(false, true) => &FOR_V1_BLOCKED,
126+
(true, false) => &FOR_V2,
127+
(true, true) => &FOR_V2_BLOCKED,
105128
}
106129
}
107130

@@ -214,10 +237,12 @@ impl Scheme for FoRScheme {
214237
// of bitpacking.
215238
// NOTE: we could delegate in the future if we had another downstream codec that performs
216239
// as well.
217-
let leaf_ctx = compress_ctx.clone().as_leaf();
218240
let biased_data =
219241
ArrayAndStats::new(biased.into_array(), compress_ctx.merged_stats_options());
220-
let compressed = BITPACKING_V1.compress(compressor, &biased_data, leaf_ctx, exec_ctx)?;
242+
// Our own context, not a leaf one, so a v2 child's block offsets can cascade.
243+
let compressed =
244+
self.bitpacking
245+
.compress(compressor, &biased_data, compress_ctx.clone(), exec_ctx)?;
221246

222247
// TODO(connor): This should really be `new_unchecked`.
223248
let for_compressed = match for_array.references() {
@@ -241,9 +266,11 @@ impl Scheme for FoRScheme {
241266
mod tests {
242267
use rstest::rstest;
243268
use vortex_array::ArrayId;
269+
use vortex_fastlanes::bitpacked_v2_id;
244270
use vortex_fastlanes::for_v1_id;
245271
use vortex_fastlanes::for_v2_id;
246272

273+
use super::BitPackingScheme;
247274
use super::FOR_V1;
248275
use super::FOR_V2;
249276
use crate::Scheme;
@@ -265,11 +292,26 @@ mod tests {
265292
|id: &ArrayId| (allow_v1 && *id == for_v1_id()) || (allow_v2 && *id == for_v2_id());
266293
let refined = scheme.refine(&allowed);
267294
assert_eq!(refined.id(), scheme.id());
268-
let expected = if expect_v2 {
295+
let mut expected = if expect_v2 {
269296
vec![for_v1_id(), for_v2_id()]
270297
} else {
271298
vec![for_v1_id()]
272299
};
300+
expected.extend(BitPackingScheme::v1().produced_encodings());
273301
assert_eq!(refined.produced_encodings(), expected);
274302
}
303+
304+
/// Both variants give the child per-block bit widths exactly when `fastlanes.bitpacked.v2` is
305+
/// allowed.
306+
#[rstest]
307+
fn refine_picks_blocked_child_when_bitpacked_v2_is_allowed(
308+
#[values(false, true)] allow_v2: bool,
309+
#[values(&FOR_V1, &FOR_V2)] scheme: &'static dyn Scheme,
310+
) {
311+
let refined = scheme.refine(&|id: &ArrayId| allow_v2 && *id == bitpacked_v2_id());
312+
assert_eq!(
313+
refined.produced_encodings().contains(&bitpacked_v2_id()),
314+
allow_v2
315+
);
316+
}
275317
}

‎vortex-btrblocks/tests/for_config.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use vortex_edition::EditionSession;
2727
use vortex_edition::EditionSessionExt;
2828
use vortex_edition::declarations::core::CORE_2026_08_3;
2929
use vortex_error::VortexResult;
30+
use vortex_fastlanes::bitpacked_v2_id;
3031
use vortex_fastlanes::for_v1_id;
3132
use vortex_fastlanes::for_v2_id;
3233
use vortex_session::VortexSession;
@@ -79,6 +80,12 @@ fn clustered() -> ArrayRef {
7980
PrimitiveArray::from_iter((0..8192u32).map(|i| 1_000_000 + i % 100)).into_array()
8081
}
8182

83+
/// Values whose range doubles every chunk from the same base, so the child needs a different bit
84+
/// width in each block.
85+
fn widening() -> ArrayRef {
86+
PrimitiveArray::from_iter((0..8192u32).map(|i| 1_000_000 + i % (4 << (i / 1024)))).into_array()
87+
}
88+
8289
/// Compresses `array`, round trips it through serialization, and returns the serialized IDs.
8390
fn compress_roundtrip(
8491
builder: BtrBlocksCompressorBuilder,
@@ -167,3 +174,17 @@ fn cuda_preset_keeps_single_reference() -> VortexResult<()> {
167174
assert!(!ids.contains(&for_v2_id()));
168175
Ok(())
169176
}
177+
178+
/// FoR bit-packs its child with per-block widths exactly when `fastlanes.bitpacked.v2` is allowed.
179+
#[rstest]
180+
#[case::all_allowed(for_only(), true)]
181+
#[case::cuda(for_only().only_cuda_compatible(), false)]
182+
fn child_widths_follow_bitpacked_v2(
183+
#[case] builder: BtrBlocksCompressorBuilder,
184+
#[case] expect_v2: bool,
185+
) -> VortexResult<()> {
186+
let ids = compress_roundtrip(builder, &widening())?;
187+
assert!(ids.contains(&for_v1_id()));
188+
assert_eq!(ids.contains(&bitpacked_v2_id()), expect_v2);
189+
Ok(())
190+
}

‎vortex-file/src/tests.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1698,12 +1698,11 @@ async fn test_into_tokio_array_stream() -> VortexResult<()> {
16981698
#[tokio::test]
16991699
async fn test_array_stream_no_double_dict_encode() -> VortexResult<()> {
17001700
let num_vals = 2048;
1701-
let mut values = Vec::<i64>::with_capacity(num_vals);
1702-
// Far apart, so per-block bit widths can't pack them tighter than a dictionary.
1703-
values.extend(iter::repeat_n(1_000_000_000, num_vals / 2));
1704-
values.extend(iter::repeat_n(2_000_000_000, num_vals / 2));
1705-
1706-
let array = PrimitiveArray::from_iter(values).into_array();
1701+
// Alternating and far apart, so neither runs nor bit widths pack them tighter than a
1702+
// dictionary.
1703+
let array =
1704+
PrimitiveArray::from_iter((0..num_vals).map(|i| [1_000_000_000i64, 2_000_000_000][i % 2]))
1705+
.into_array();
17071706
let mut buf = Vec::new();
17081707
SESSION
17091708
.write_options()

0 commit comments

Comments
 (0)