Skip to content

Commit c6fc22e

Browse files
committed
Only compress with encodings the session can write; add BtrBlocksCompressor::new
Signed-off-by: Matt Katz <[email protected]>
1 parent 033636e commit c6fc22e

16 files changed

Lines changed: 243 additions & 180 deletions

‎vortex-btrblocks/benches/compress.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ mod benchmarks {
2020
use vortex_array::VortexSessionExecute;
2121
use vortex_array::arrays::PrimitiveArray;
2222
use vortex_btrblocks::BtrBlocksCompressor;
23-
use vortex_btrblocks::BtrBlocksOptions;
23+
use vortex_btrblocks::CompressionSession;
2424
use vortex_buffer::buffer_mut;
2525
use vortex_session::VortexSession;
2626
use vortex_utils::aliases::hash_set::HashSet;
@@ -52,13 +52,7 @@ mod benchmarks {
5252
let array = make_clickbench_window_name()
5353
.execute::<PrimitiveArray>(&mut ctx)
5454
.unwrap();
55-
let compressor = BtrBlocksCompressor::from_session_with_options(
56-
&SESSION,
57-
&BtrBlocksOptions {
58-
enforce_editions: false,
59-
..Default::default()
60-
},
61-
);
55+
let compressor = BtrBlocksCompressor::new(CompressionSession::default().schemes().to_vec());
6256
bencher
6357
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
6458
.input_counter(|(array, _)| ItemsCount::new(array.len()))

‎vortex-btrblocks/benches/compress_listview.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ mod benchmarks {
2525
use vortex_array::dtype::FieldNames;
2626
use vortex_array::validity::Validity;
2727
use vortex_btrblocks::BtrBlocksCompressor;
28-
use vortex_btrblocks::BtrBlocksOptions;
28+
use vortex_btrblocks::CompressionSession;
2929
use vortex_buffer::buffer_mut;
3030
use vortex_session::VortexSession;
3131

@@ -184,13 +184,7 @@ mod benchmarks {
184184
fn compress_listview(bencher: Bencher, layout: OffsetLayout) {
185185
let array = build_nested_listview(NUM_ROWS, layout);
186186
let nbytes = array.nbytes();
187-
let compressor = BtrBlocksCompressor::from_session_with_options(
188-
&SESSION,
189-
&BtrBlocksOptions {
190-
enforce_editions: false,
191-
..Default::default()
192-
},
193-
);
187+
let compressor = BtrBlocksCompressor::new(CompressionSession::default().schemes().to_vec());
194188
bencher
195189
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
196190
.input_counter(|_| ItemsCount::new(NUM_ROWS))

‎vortex-btrblocks/src/canonical_compressor.rs‎

Lines changed: 82 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,28 @@
55
66
use std::ops::Deref;
77

8+
use vortex_array::ArrayId;
89
use vortex_array::ArrayRef;
910
use vortex_array::ExecutionCtx;
11+
use vortex_array::session::ArraySessionExt;
12+
use vortex_edition::ComponentKind;
13+
use vortex_edition::EditionSessionExt;
1014
use vortex_error::VortexResult;
1115
use vortex_session::VortexSession;
16+
use vortex_utils::aliases::hash_set::HashSet;
1217

1318
use crate::CascadingCompressor;
1419
use crate::CompressionSessionExt;
20+
use crate::Scheme;
1521
use crate::SchemeExt;
1622
use crate::SchemeId;
1723

1824
/// Options for building a [`BtrBlocksCompressor`] from a session.
1925
#[derive(Clone, Debug)]
2026
pub struct BtrBlocksOptions {
21-
/// Keep only the registered schemes whose serialized IDs the session's enabled editions
22-
/// permit, which a file writer requires. Off, every registered scheme is used, for in-memory
23-
/// compression where no edition applies.
27+
/// Keep only the schemes whose serialized IDs the session's enabled editions permit, which a
28+
/// file writer requires. Off, for in-memory compression where no edition applies, every
29+
/// scheme whose encodings have a registered plugin is used.
2430
pub enforce_editions: bool,
2531
/// Schemes to leave out.
2632
pub exclude_schemes: Vec<SchemeId>,
@@ -66,9 +72,17 @@ pub struct BtrBlocksCompressor(
6672
);
6773

6874
impl BtrBlocksCompressor {
75+
/// Creates a compressor over `schemes` as given.
76+
///
77+
/// Prefer [`from_session`](Self::from_session) for writing files: it only keeps schemes the
78+
/// session can serialize and its editions permit, which this constructor does not check.
79+
pub fn new(schemes: Vec<&'static dyn Scheme>) -> Self {
80+
Self(CascadingCompressor::new(schemes))
81+
}
82+
6983
/// A compressor with no schemes, which leaves every array as it is.
7084
pub fn empty() -> Self {
71-
Self(CascadingCompressor::new(Vec::new()))
85+
Self::new(Vec::new())
7286
}
7387

7488
/// Creates a compressor over the schemes registered on `session` whose serialized IDs the
@@ -78,14 +92,14 @@ impl BtrBlocksCompressor {
7892
}
7993

8094
/// Creates a compressor over the schemes registered on `session`, per `options`.
95+
///
96+
/// Only schemes the session can write are kept: every encoding they produce has a registered
97+
/// array plugin and, with `enforce_editions`, is permitted by the enabled editions. This is
98+
/// the rule the file writer applies to the arrays it serializes.
8199
pub fn from_session_with_options(session: &VortexSession, options: &BtrBlocksOptions) -> Self {
82-
let mut schemes = if options.enforce_editions {
83-
session.permitted_schemes()
84-
} else {
85-
session.registered_schemes()
86-
};
100+
let mut schemes = writable_schemes(session, options.enforce_editions);
87101
schemes.retain(|scheme| !options.exclude_schemes.contains(&scheme.id()));
88-
Self(CascadingCompressor::new(schemes))
102+
Self::new(schemes)
89103
}
90104

91105
/// Compresses an array using BtrBlocks-inspired compression.
@@ -94,10 +108,68 @@ impl BtrBlocksCompressor {
94108
}
95109
}
96110

111+
/// The registered schemes whose produced encodings all have a registered array plugin and, with
112+
/// `enforce_editions`, are permitted by the enabled editions.
113+
fn writable_schemes(session: &VortexSession, enforce_editions: bool) -> Vec<&'static dyn Scheme> {
114+
let registered: HashSet<ArrayId> = session
115+
.arrays()
116+
.registry()
117+
.read(|registry| registry.keys().copied().collect());
118+
let allowed: HashSet<ArrayId> = if enforce_editions {
119+
session
120+
.enabled_component_ids(ComponentKind::Array)
121+
.into_iter()
122+
.filter(|id| registered.contains(id))
123+
.collect()
124+
} else {
125+
registered
126+
};
127+
session
128+
.compression()
129+
.schemes()
130+
.iter()
131+
.copied()
132+
.filter(|scheme| {
133+
scheme
134+
.produced_encodings()
135+
.iter()
136+
.all(|id| allowed.contains(id))
137+
})
138+
.collect()
139+
}
140+
97141
impl Deref for BtrBlocksCompressor {
98142
type Target = CascadingCompressor;
99143

100144
fn deref(&self) -> &CascadingCompressor {
101145
&self.0
102146
}
103147
}
148+
149+
#[cfg(test)]
150+
mod tests {
151+
use vortex_array::array_session;
152+
153+
use super::*;
154+
use crate::SchemeId;
155+
use crate::schemes::integer::FoRScheme;
156+
use crate::schemes::integer::IntDictScheme;
157+
158+
fn ids(schemes: &[&'static dyn Scheme]) -> Vec<SchemeId> {
159+
schemes.iter().map(|scheme| scheme.id()).collect()
160+
}
161+
162+
/// Without enabled editions no serialized ID is permitted, so nothing survives.
163+
#[test]
164+
fn no_editions_permit_nothing() {
165+
assert!(writable_schemes(&array_session(), true).is_empty());
166+
}
167+
168+
/// A scheme whose encoding has no registered plugin can never be written, editions or not.
169+
#[test]
170+
fn unregistered_encodings_are_never_writable() {
171+
let writable = ids(&writable_schemes(&array_session(), false));
172+
assert!(writable.contains(&IntDictScheme.id()));
173+
assert!(!writable.contains(&FoRScheme.id()));
174+
}
175+
}

‎vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,15 @@ use vortex_buffer::Buffer;
1919
use vortex_error::VortexResult;
2020
use vortex_session::VortexSession;
2121

22-
use crate::tests::no_editions_compressor;
22+
use crate::tests::default_compressor;
2323

2424
static SESSION: LazyLock<VortexSession> = LazyLock::new(vortex_array::array_session);
2525

2626
#[test]
2727
fn test_constant_compressed() -> VortexResult<()> {
2828
let values: Vec<f64> = vec![42.5; 100];
2929
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
30-
let btr = no_editions_compressor(&SESSION);
30+
let btr = default_compressor();
3131
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
3232
assert!(compressed.is::<Constant>());
3333
Ok(())
@@ -37,7 +37,7 @@ fn test_constant_compressed() -> VortexResult<()> {
3737
fn test_alp_compressed() -> VortexResult<()> {
3838
let values: Vec<f64> = (0..1000).map(|i| (i as f64) * 0.01).collect();
3939
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
40-
let btr = no_editions_compressor(&SESSION);
40+
let btr = default_compressor();
4141
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
4242
assert!(compressed.is::<ALP>());
4343
Ok(())
@@ -50,7 +50,7 @@ fn test_dict_compressed() -> VortexResult<()> {
5050
.map(|i| distinct_values[i % distinct_values.len()])
5151
.collect();
5252
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
53-
let btr = no_editions_compressor(&SESSION);
53+
let btr = default_compressor();
5454
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
5555
assert!(compressed.is::<ALP>());
5656
assert!(compressed.children()[0].is::<Dict>());
@@ -69,7 +69,7 @@ fn test_null_dominated_compressed() -> VortexResult<()> {
6969
}
7070
builder.append_nulls(95);
7171
let array = builder.finish_into_primitive();
72-
let btr = no_editions_compressor(&SESSION);
72+
let btr = default_compressor();
7373
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
7474
// Verify the compressed array preserves values.
7575
assert_eq!(compressed.len(), 100);

‎vortex-btrblocks/src/schemes/float/tests.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,18 @@ use vortex_array::dtype::Nullability;
1515
use vortex_array::validity::Validity;
1616
use vortex_buffer::Buffer;
1717
use vortex_buffer::buffer_mut;
18-
use vortex_compressor::CascadingCompressor;
1918
use vortex_error::VortexResult;
2019
use vortex_fastlanes::RLE;
2120
use vortex_session::VortexSession;
2221

22+
use crate::BtrBlocksCompressor;
2323
use crate::schemes::float::FloatRLEScheme;
24-
use crate::tests::no_editions_compressor;
24+
use crate::tests::default_compressor;
2525
static SESSION: LazyLock<VortexSession> = LazyLock::new(vortex_array::array_session);
2626

2727
#[test]
2828
fn test_empty() -> VortexResult<()> {
29-
let btr = no_editions_compressor(&SESSION);
29+
let btr = default_compressor();
3030
let array = PrimitiveArray::new(Buffer::<f32>::empty(), Validity::NonNullable).into_array();
3131
let result = btr.compress(&array, &mut SESSION.create_execution_ctx())?;
3232

@@ -42,7 +42,7 @@ fn test_compress() -> VortexResult<()> {
4242
}
4343

4444
let array = values.into_array();
45-
let btr = no_editions_compressor(&SESSION);
45+
let btr = default_compressor();
4646
let compressed = btr.compress(&array, &mut SESSION.create_execution_ctx())?;
4747
assert_eq!(compressed.len(), 1024);
4848

@@ -66,7 +66,7 @@ fn test_rle_compression() -> VortexResult<()> {
6666

6767
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
6868

69-
let compressor = CascadingCompressor::new(vec![&FloatRLEScheme]);
69+
let compressor = BtrBlocksCompressor::new(vec![&FloatRLEScheme]);
7070
let compressed =
7171
compressor.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
7272
assert!(compressed.is::<RLE>());
@@ -92,7 +92,7 @@ fn test_sparse_compression() -> VortexResult<()> {
9292
array.append_nulls(90);
9393

9494
let array = array.finish_into_primitive().into_array();
95-
let btr = no_editions_compressor(&SESSION);
95+
let btr = default_compressor();
9696
let compressed = btr.compress(&array, &mut SESSION.create_execution_ctx())?;
9797
assert_eq!(compressed.len(), 96);
9898

‎vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,23 +28,24 @@ use vortex_session::VortexSession;
2828
use vortex_sparse::Sparse;
2929

3030
use crate::BtrBlocksCompressor;
31-
use crate::CompressionSessionExt;
31+
use crate::CompressionSession;
3232
use crate::DELTA_SCHEME;
33-
use crate::tests::no_editions_compressor;
33+
use crate::tests::compressor;
34+
use crate::tests::default_compressor;
3435
static SESSION: LazyLock<VortexSession> = LazyLock::new(vortex_array::array_session);
3536

3637
/// The default schemes plus opt-in Delta.
3738
fn with_delta() -> BtrBlocksCompressor {
38-
let session = vortex_array::array_session();
39-
session.register_scheme(&DELTA_SCHEME);
40-
no_editions_compressor(&session)
39+
let mut registry = CompressionSession::default();
40+
registry.register(&DELTA_SCHEME);
41+
compressor(&registry)
4142
}
4243

4344
#[test]
4445
fn test_constant_compressed() -> VortexResult<()> {
4546
let values: Vec<i32> = iter::repeat_n(42, 100).collect();
4647
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
47-
let btr = no_editions_compressor(&SESSION);
48+
let btr = default_compressor();
4849
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
4950
assert!(compressed.is::<Constant>());
5051
Ok(())
@@ -54,7 +55,7 @@ fn test_constant_compressed() -> VortexResult<()> {
5455
fn test_for_compressed() -> VortexResult<()> {
5556
let values: Vec<i32> = (0..1000).map(|i| 1_000_000 + ((i * 37) % 100)).collect();
5657
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
57-
let btr = no_editions_compressor(&SESSION);
58+
let btr = default_compressor();
5859
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
5960
assert!(compressed.is::<FoR>());
6061
Ok(())
@@ -64,7 +65,7 @@ fn test_for_compressed() -> VortexResult<()> {
6465
fn test_bitpacking_compressed() -> VortexResult<()> {
6566
let values: Vec<u32> = (0..1000).map(|i| i % 16).collect();
6667
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
67-
let btr = no_editions_compressor(&SESSION);
68+
let btr = default_compressor();
6869
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
6970
assert!(compressed.is::<BitPacked>());
7071
assert_eq!(
@@ -93,7 +94,7 @@ fn test_sparse_compressed() -> VortexResult<()> {
9394
}
9495
}
9596
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
96-
let btr = no_editions_compressor(&SESSION);
97+
let btr = default_compressor();
9798
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
9899
assert!(compressed.is::<Sparse>());
99100
Ok(())
@@ -117,7 +118,7 @@ fn test_dict_compressed() -> VortexResult<()> {
117118
}
118119

119120
let array = PrimitiveArray::new(Buffer::copy_from(&codes), Validity::NonNullable);
120-
let btr = no_editions_compressor(&SESSION);
121+
let btr = default_compressor();
121122
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
122123
assert!(compressed.is::<Dict>());
123124
Ok(())
@@ -130,7 +131,7 @@ fn test_runend_compressed() -> VortexResult<()> {
130131
values.extend(iter::repeat_n((i32::MAX - 50).wrapping_add(i), 10));
131132
}
132133
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
133-
let btr = no_editions_compressor(&SESSION);
134+
let btr = default_compressor();
134135
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
135136
assert!(compressed.is::<RunEnd>());
136137
Ok(())
@@ -140,7 +141,7 @@ fn test_runend_compressed() -> VortexResult<()> {
140141
fn test_sequence_compressed() -> VortexResult<()> {
141142
let values: Vec<i32> = (0..1000).map(|i| i * 7).collect();
142143
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
143-
let btr = no_editions_compressor(&SESSION);
144+
let btr = default_compressor();
144145
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
145146
assert!(compressed.is::<Sequence>());
146147
Ok(())
@@ -157,7 +158,7 @@ fn test_rle_compressed() -> VortexResult<()> {
157158
values.extend(iter::repeat_n(v, 10));
158159
}
159160
let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
160-
let btr = no_editions_compressor(&SESSION);
161+
let btr = default_compressor();
161162
let compressed = btr.compress(&array.into_array(), &mut SESSION.create_execution_ctx())?;
162163
eprintln!("{}", compressed.display_tree());
163164
assert!(compressed.is::<RunEnd>());

0 commit comments

Comments
 (0)