Skip to content

Commit 80af2c5

Browse files
committed
Register compression schemes on the session
Signed-off-by: Matt Katz <[email protected]>
1 parent 2d9414a commit 80af2c5

44 files changed

Lines changed: 761 additions & 322 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/compress-bench/src/gpu/vortex.rs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ use vortex_bench::compress::CompressedData;
3737
use vortex_bench::compress::Compressor;
3838
use vortex_bench::compress::Uncompressed;
3939
use vortex_bench::conversions::parquet_to_vortex_chunks_with_batch_size;
40-
use vortex_bench::retain_edition_encodings;
4140
use vortex_cuda::CanonicalCudaExt;
4241
use vortex_cuda::CudaExecutionCtx;
4342
use vortex_cuda::CudaOpenOptionsExt;
@@ -100,11 +99,9 @@ impl Compressor for GpuVortexCompressor {
10099
// partition rather than whatever the default strategy would regroup them into.
101100
let strategy = Arc::new(ChunkedLayoutStrategy::new(CompressingStrategy::new(
102101
CudaFlatLayoutStrategy::default(),
103-
retain_edition_encodings(
104-
&SESSION,
105-
BtrBlocksCompressorBuilder::default().only_cuda_compatible(),
106-
)
107-
.build(),
102+
BtrBlocksCompressorBuilder::from_session(&SESSION)
103+
.only_cuda_compatible()
104+
.build(),
108105
)));
109106
let start = Instant::now();
110107
SESSION

‎benchmarks/string-bench/src/serialized.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ use crate::prepare_column;
5757
use crate::throughput;
5858
use crate::verify_canonicalized;
5959

60-
/// The btrblocks string schemes that `BtrBlocksCompressorBuilder::default()` can
60+
/// The btrblocks string schemes that `BtrBlocksCompressorBuilder::from_session` can
6161
/// choose between. Forcing one encoder excludes every entry except its own
6262
/// scheme, so this list must track the default scheme set: add a row whenever a
6363
/// new string encoder becomes selectable by default (e.g. Zstd).
@@ -170,15 +170,18 @@ impl SerializedResult {
170170

171171
/// Build the file writer strategy that forces one selected string scheme while
172172
/// leaving editioned non-string child compression enabled.
173-
fn serialized_write_strategy(encoder: StringEncoder) -> Arc<dyn LayoutStrategy> {
173+
fn serialized_write_strategy(
174+
session: &VortexSession,
175+
encoder: StringEncoder,
176+
) -> Arc<dyn LayoutStrategy> {
174177
let forced = encoder.scheme_id();
175-
let compressor = BtrBlocksCompressorBuilder::default().exclude_schemes(
178+
let compressor = BtrBlocksCompressorBuilder::from_session(session).exclude_schemes(
176179
default_string_scheme_ids()
177180
.into_iter()
178181
.filter(|&id| id != forced)
179182
.chain([DeltaScheme::default().id()]),
180183
);
181-
WriteStrategyBuilder::default()
184+
WriteStrategyBuilder::from_session(session)
182185
.with_btrblocks_builder(compressor)
183186
.build()
184187
}
@@ -251,7 +254,7 @@ async fn prepare_serialized_file(
251254
verify: bool,
252255
ctx: &mut ExecutionCtx,
253256
) -> Result<SerializedFile> {
254-
let strategy = serialized_write_strategy(encoder);
257+
let strategy = serialized_write_strategy(session, encoder);
255258
let data = write_serialized_file(session, input, &strategy).await?;
256259
let file_bytes = data.len() as u64;
257260

‎encodings/parquet-variant/src/vtable.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -466,8 +466,11 @@ mod tests {
466466
}
467467

468468
#[fixture]
469-
fn write_strategy() -> Arc<dyn LayoutStrategy> {
470-
vortex_file::WriteStrategyBuilder::default().build()
469+
fn write_strategy(
470+
parquet_variant_file_session: VortexResult<VortexSession>,
471+
) -> VortexResult<Arc<dyn LayoutStrategy>> {
472+
let session = parquet_variant_file_session?;
473+
Ok(vortex_file::WriteStrategyBuilder::from_session(&session).build())
471474
}
472475

473476
#[test]
@@ -544,15 +547,15 @@ mod tests {
544547
async fn test_file_roundtrip_typed_value_variant_with_zoned_strategy(
545548
#[from(typed_value_variant_array)] expected: VortexResult<ArrayRef>,
546549
parquet_variant_file_session: VortexResult<VortexSession>,
547-
write_strategy: Arc<dyn LayoutStrategy>,
550+
write_strategy: VortexResult<Arc<dyn LayoutStrategy>>,
548551
) -> VortexResult<()> {
549552
let expected = expected?;
550553
let parquet_variant_file_session = parquet_variant_file_session?;
551554

552555
let mut bytes = ByteBufferMut::empty();
553556
parquet_variant_file_session
554557
.write_options()
555-
.with_strategy(write_strategy)
558+
.with_strategy(write_strategy?)
556559
.write(&mut bytes, expected.to_array_stream())
557560
.await?;
558561

‎fuzz/fuzz_targets/file_io.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,10 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus {
6565
let write_options = match compressor_strategy {
6666
CompressorStrategy::Default => SESSION.write_options(),
6767
CompressorStrategy::Compact => SESSION.write_options().with_strategy(
68-
WriteStrategyBuilder::default()
69-
.with_btrblocks_builder(BtrBlocksCompressorBuilder::default().with_compact())
68+
WriteStrategyBuilder::from_session(&SESSION)
69+
.with_btrblocks_builder(
70+
BtrBlocksCompressorBuilder::from_session(&SESSION).with_compact(),
71+
)
7072
.build(),
7173
),
7274
};

‎fuzz/src/array/mod.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ use vortex_array::scalar_fn::fns::operators::Operator;
6464
use vortex_array::search_sorted::SearchResult;
6565
use vortex_array::search_sorted::SearchSorted;
6666
use vortex_array::search_sorted::SearchSortedSide;
67-
use vortex_btrblocks::BtrBlocksCompressor;
68-
#[cfg(feature = "zstd")]
6967
use vortex_btrblocks::BtrBlocksCompressorBuilder;
7068
use vortex_error::VortexExpect;
7169
use vortex_error::vortex_panic;
@@ -249,7 +247,8 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction {
249247
.into_array()
250248
};
251249

252-
let compressed = BtrBlocksCompressor::default()
250+
let compressed = BtrBlocksCompressorBuilder::from_session(&SESSION)
251+
.build()
253252
.compress(&indices_array, &mut ctx)
254253
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test");
255254
(
@@ -561,10 +560,11 @@ pub fn compress_array(
561560
ctx: &mut ExecutionCtx,
562561
) -> ArrayRef {
563562
match strategy {
564-
CompressorStrategy::Default => BtrBlocksCompressor::default()
563+
CompressorStrategy::Default => BtrBlocksCompressorBuilder::from_session(&SESSION)
564+
.build()
565565
.compress(array, ctx)
566566
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test"),
567-
CompressorStrategy::Compact => BtrBlocksCompressorBuilder::default()
567+
CompressorStrategy::Compact => BtrBlocksCompressorBuilder::from_session(&SESSION)
568568
.with_compact()
569569
.build()
570570
.compress(array, ctx)
@@ -579,7 +579,8 @@ pub fn compress_array(
579579
_strategy: CompressorStrategy,
580580
ctx: &mut ExecutionCtx,
581581
) -> ArrayRef {
582-
BtrBlocksCompressor::default()
582+
BtrBlocksCompressorBuilder::from_session(&SESSION)
583+
.build()
583584
.compress(array, ctx)
584585
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test")
585586
}

‎vortex-bench/src/conversions.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ use wkb::writer::write_geometry;
6666
use crate::CompactionStrategy;
6767
use crate::Format;
6868
use crate::SESSION;
69-
use crate::retain_edition_encodings;
7069
use crate::utils::file::idempotent_async;
7170

7271
/// Memory budget per concurrent conversion stream in GB. This is somewhat arbitary.
@@ -246,12 +245,11 @@ fn write_options_for(
246245
return compaction.apply_options(SESSION.write_options());
247246
}
248247

249-
let mut builder = WriteStrategyBuilder::default();
248+
let mut builder = WriteStrategyBuilder::from_session(&SESSION);
250249
if matches!(compaction, CompactionStrategy::Compact) {
251-
builder = builder.with_btrblocks_builder(retain_edition_encodings(
252-
&SESSION,
253-
BtrBlocksCompressorBuilder::default().with_compact(),
254-
));
250+
builder = builder.with_btrblocks_builder(
251+
BtrBlocksCompressorBuilder::from_session(&SESSION).with_compact(),
252+
);
255253
}
256254
for name in binary_fields {
257255
builder = builder.with_field_writer(FieldPath::from_name(name), no_dict_layout());
@@ -263,7 +261,7 @@ fn write_options_for(
263261
fn no_dict_layout() -> Arc<dyn LayoutStrategy> {
264262
Arc::new(CompressingStrategy::new(
265263
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
266-
retain_edition_encodings(&SESSION, BtrBlocksCompressorBuilder::default()).build(),
264+
BtrBlocksCompressorBuilder::from_session(&SESSION).build(),
267265
))
268266
}
269267

‎vortex-bench/src/lib.rs‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,6 @@ pub use datasets::BenchmarkDataset;
7070
pub use output::BenchmarkOutput;
7171
pub use output::create_output_writer;
7272
use vortex::VortexSessionDefault;
73-
use vortex::editions::ComponentKind;
74-
use vortex::editions::EditionSessionExt;
7573
pub use vortex::error::vortex_panic;
7674
use vortex::io::session::RuntimeSessionExt;
7775
use vortex::session::VortexSession;
@@ -254,33 +252,17 @@ impl CompactionStrategy {
254252
pub fn apply_options(&self, options: VortexWriteOptions) -> VortexWriteOptions {
255253
match self {
256254
CompactionStrategy::Compact => options.with_strategy(
257-
WriteStrategyBuilder::default()
258-
.with_btrblocks_builder(retain_edition_encodings(
259-
&SESSION,
260-
BtrBlocksCompressorBuilder::default().with_compact(),
261-
))
255+
WriteStrategyBuilder::from_session(&SESSION)
256+
.with_btrblocks_builder(
257+
BtrBlocksCompressorBuilder::from_session(&SESSION).with_compact(),
258+
)
262259
.build(),
263260
),
264261
CompactionStrategy::Default => options,
265262
}
266263
}
267264
}
268265

269-
/// Restrict `builder` to the encodings permitted by the session's enabled editions.
270-
///
271-
/// The default writer applies this filter itself. An explicit strategy bypasses it, so a
272-
/// benchmark that builds its own compressor applies it here to stay within editions.
273-
pub fn retain_edition_encodings(
274-
session: &VortexSession,
275-
builder: BtrBlocksCompressorBuilder,
276-
) -> BtrBlocksCompressorBuilder {
277-
let allowed = session
278-
.enabled_component_ids(ComponentKind::Array)
279-
.into_iter()
280-
.collect();
281-
builder.retain_allowed_encodings(&allowed)
282-
}
283-
284266
/// Verify that local data has already been prepared for the requested benchmark formats.
285267
///
286268
/// Engine-specific benchmark binaries call this before running queries. Data generation itself

‎vortex-btrblocks/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,15 @@ vortex-buffer = { workspace = true }
2626
vortex-compressor = { workspace = true }
2727
vortex-datetime-parts = { workspace = true }
2828
vortex-decimal-byte-parts = { workspace = true }
29+
vortex-edition = { workspace = true }
2930
vortex-error = { workspace = true }
3031
vortex-fastlanes = { workspace = true }
3132
vortex-fsst = { workspace = true }
3233
vortex-onpair = { workspace = true }
3334
vortex-pco = { workspace = true, optional = true }
3435
vortex-runend = { workspace = true }
3536
vortex-sequence = { workspace = true }
37+
vortex-session = { workspace = true }
3638
vortex-sparse = { workspace = true }
3739
vortex-utils = { workspace = true }
3840
vortex-zigzag = { workspace = true }
@@ -49,9 +51,7 @@ tpchgen = { workspace = true }
4951
tpchgen-arrow = { workspace = true }
5052
vortex-array = { workspace = true, features = ["_test-harness"] }
5153
vortex-arrow = { workspace = true }
52-
vortex-edition = { workspace = true }
5354
vortex-mask = { workspace = true }
54-
vortex-session = { workspace = true }
5555

5656
[features]
5757
pco = ["dep:pco", "dep:vortex-pco"]

‎vortex-btrblocks/benches/compress.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ mod benchmarks {
1919
use vortex_array::IntoArray;
2020
use vortex_array::VortexSessionExecute;
2121
use vortex_array::arrays::PrimitiveArray;
22-
use vortex_btrblocks::BtrBlocksCompressor;
22+
use vortex_btrblocks::BtrBlocksCompressorBuilder;
2323
use vortex_buffer::buffer_mut;
2424
use vortex_session::VortexSession;
2525
use vortex_utils::aliases::hash_set::HashSet;
@@ -51,7 +51,9 @@ mod benchmarks {
5151
let array = make_clickbench_window_name()
5252
.execute::<PrimitiveArray>(&mut ctx)
5353
.unwrap();
54-
let compressor = BtrBlocksCompressor::default();
54+
let compressor = BtrBlocksCompressorBuilder::from_session(&SESSION)
55+
.allow_all_encodings()
56+
.build();
5557
bencher
5658
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
5759
.input_counter(|(array, _)| ItemsCount::new(array.len()))

‎vortex-btrblocks/benches/compress_listview.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ mod benchmarks {
2424
use vortex_array::arrays::VarBinViewArray;
2525
use vortex_array::dtype::FieldNames;
2626
use vortex_array::validity::Validity;
27-
use vortex_btrblocks::BtrBlocksCompressor;
27+
use vortex_btrblocks::BtrBlocksCompressorBuilder;
2828
use vortex_buffer::buffer_mut;
2929
use vortex_session::VortexSession;
3030

@@ -183,7 +183,9 @@ mod benchmarks {
183183
fn compress_listview(bencher: Bencher, layout: OffsetLayout) {
184184
let array = build_nested_listview(NUM_ROWS, layout);
185185
let nbytes = array.nbytes();
186-
let compressor = BtrBlocksCompressor::default();
186+
let compressor = BtrBlocksCompressorBuilder::from_session(&SESSION)
187+
.allow_all_encodings()
188+
.build();
187189
bencher
188190
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
189191
.input_counter(|_| ItemsCount::new(NUM_ROWS))

0 commit comments

Comments
 (0)