Skip to content

Commit fb3b160

Browse files
committed
add ZonedStrategyBuilder with per-field zone maps
Signed-off-by: Matt Katz <[email protected]>
1 parent 9c12160 commit fb3b160

9 files changed

Lines changed: 572 additions & 168 deletions

File tree

‎vortex-file/src/strategy.rs‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ use vortex_layout::layouts::list::writer::ListLayoutStrategy;
2727
use vortex_layout::layouts::repartition::RepartitionStrategy;
2828
use vortex_layout::layouts::repartition::RepartitionWriterOptions;
2929
use vortex_layout::layouts::table::TableStrategy;
30-
use vortex_layout::layouts::zoned::writer::ZonedLayoutOptions;
31-
use vortex_layout::layouts::zoned::writer::ZonedStrategy;
30+
use vortex_layout::layouts::zoned::writer::ZoneOptions;
31+
use vortex_layout::layouts::zoned::writer::ZonedStrategyBuilder;
3232
use vortex_session::VortexSession;
3333
use vortex_utils::aliases::hash_map::HashMap;
3434

@@ -66,6 +66,7 @@ pub struct WriteStrategyBuilder {
6666
///
6767
/// [`ListLayoutStrategy`]: vortex_layout::layouts::list::writer::ListLayoutStrategy
6868
use_list_layout: bool,
69+
zoned: Option<ZonedStrategyBuilder>,
6970
}
7071

7172
impl WriteStrategyBuilder {
@@ -84,10 +85,12 @@ impl WriteStrategyBuilder {
8485
use_list_layout: session
8586
.enabled_component_ids(ComponentKind::Layout)
8687
.contains(&List.id()),
88+
zoned: None,
8789
}
8890
}
8991

90-
/// Override the row block size used for row repartitioning and zoned statistics.
92+
/// Override the row block size used for row repartitioning and, unless [`Self::with_zoned`]
93+
/// is set, the zone length of zoned statistics.
9194
///
9295
/// Larger blocks reduce footer/statistics overhead. Smaller blocks can improve pruning and
9396
/// random-access locality.
@@ -96,6 +99,16 @@ impl WriteStrategyBuilder {
9699
self
97100
}
98101

102+
/// Override the zone maps written for each column.
103+
///
104+
/// By default every column gets one zone map of [`Self::with_row_block_size`] rows holding the
105+
/// default pruning aggregates. Field paths are matched after the root struct is split into
106+
/// columns, as for [`Self::with_field_writer`].
107+
pub fn with_zoned(mut self, zoned: ZonedStrategyBuilder) -> Self {
108+
self.zoned = Some(zoned);
109+
self
110+
}
111+
99112
/// Override the target uncompressed byte size used to coalesce data blocks.
100113
///
101114
/// Passing `None` disables byte-size coalescing, so blocks retain the row granularity set by
@@ -221,16 +234,12 @@ impl WriteStrategyBuilder {
221234
);
222235

223236
let row_block_size = NonZeroUsize::new(self.row_block_size).vortex_expect("must be non 0");
237+
let zoned = self.zoned.unwrap_or_else(|| {
238+
ZonedStrategyBuilder::new().with_zones([ZoneOptions::new(row_block_size)])
239+
});
224240

225241
// 2. calculate stats for each row group
226-
let stats = ZonedStrategy::new(
227-
dict,
228-
compress_then_flat.clone(),
229-
ZonedLayoutOptions {
230-
block_size: row_block_size,
231-
..Default::default()
232-
},
233-
);
242+
let stats = zoned.clone().build(dict, compress_then_flat.clone());
234243

235244
// 1. repartition each column to fixed row counts
236245
let repartition = RepartitionStrategy::new(
@@ -257,14 +266,7 @@ impl WriteStrategyBuilder {
257266
// We need a closure here to enable recursive application of list layout.
258267
table_strategy = table_strategy.with_list_layout_factory(
259268
move |list_layout: ListLayoutStrategy| -> Arc<dyn LayoutStrategy> {
260-
let zoned = ZonedStrategy::new(
261-
list_layout,
262-
compress_then_flat.clone(),
263-
ZonedLayoutOptions {
264-
block_size: row_block_size,
265-
..Default::default()
266-
},
267-
);
269+
let zoned = zoned.clone().build(list_layout, compress_then_flat.clone());
268270
Arc::new(RepartitionStrategy::new(
269271
zoned,
270272
RepartitionWriterOptions {

‎vortex-layout/src/layouts/indexed/tests.rs‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ use crate::layouts::indexed::tests::fixed_superset::FixedSupersetIndex;
6262
use crate::layouts::indexed::tests::fixed_superset::INDEX_ROWS;
6363
use crate::layouts::repartition::RepartitionStrategy;
6464
use crate::layouts::repartition::RepartitionWriterOptions;
65-
use crate::layouts::zoned::writer::ZonedLayoutOptions;
66-
use crate::layouts::zoned::writer::ZonedStrategy;
65+
use crate::layouts::zoned::writer::ZoneOptions;
66+
use crate::layouts::zoned::writer::ZonedStrategyBuilder;
6767
use crate::scan::scan_builder::ScanBuilder;
6868
use crate::segments::SegmentFuture;
6969
use crate::segments::SegmentId;
@@ -853,14 +853,14 @@ async fn probing_loads_only_the_index_chunks_its_filter_cannot_prune() -> Vortex
853853
let session = new_session();
854854
// Two index rows per chunk, with a zone per chunk.
855855
let index = RepartitionStrategy::new(
856-
ZonedStrategy::new(
857-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
858-
FlatLayoutStrategy::default(),
859-
ZonedLayoutOptions {
860-
block_size: NonZeroUsize::new(2).ok_or_else(|| vortex_err!("non-zero"))?,
861-
..Default::default()
862-
},
863-
),
856+
ZonedStrategyBuilder::new()
857+
.with_zones([ZoneOptions::new(
858+
NonZeroUsize::new(2).ok_or_else(|| vortex_err!("non-zero"))?,
859+
)])
860+
.build(
861+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
862+
FlatLayoutStrategy::default(),
863+
),
864864
RepartitionWriterOptions {
865865
block_size_minimum: 0,
866866
block_len_multiple: 2,

‎vortex-layout/src/layouts/struct_/reader.rs‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -646,8 +646,8 @@ mod tests {
646646
use crate::layouts::chunked::writer::ChunkedLayoutStrategy;
647647
use crate::layouts::flat::writer::FlatLayoutStrategy;
648648
use crate::layouts::table::TableStrategy;
649-
use crate::layouts::zoned::writer::ZonedLayoutOptions;
650-
use crate::layouts::zoned::writer::ZonedStrategy;
649+
use crate::layouts::zoned::writer::ZoneOptions;
650+
use crate::layouts::zoned::writer::ZonedStrategyBuilder;
651651
use crate::segments::SegmentSource;
652652
use crate::segments::TestSegments;
653653
use crate::sequence::SequenceId;
@@ -1689,14 +1689,12 @@ mod tests {
16891689
let ctx = ArrayContext::empty();
16901690
let segments = Arc::new(TestSegments::default());
16911691
let (ptr, eof) = SequenceId::root().split();
1692-
let zoned = ZonedStrategy::new(
1693-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
1694-
FlatLayoutStrategy::default(),
1695-
ZonedLayoutOptions {
1696-
block_size: NonZeroUsize::new(3).unwrap(),
1697-
..Default::default()
1698-
},
1699-
);
1692+
let zoned = ZonedStrategyBuilder::new()
1693+
.with_zones([ZoneOptions::new(NonZeroUsize::new(3).unwrap())])
1694+
.build(
1695+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
1696+
FlatLayoutStrategy::default(),
1697+
);
17001698
let strategy = TableStrategy::new(Arc::new(FlatLayoutStrategy::default()), Arc::new(zoned));
17011699
let chunk = |a: [i32; 3], b: [i32; 3]| {
17021700
StructArray::from_fields(

‎vortex-layout/src/layouts/struct_/writer.rs‎

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,11 @@ impl LayoutStrategy for StructStrategy {
236236
} else {
237237
ctx.clone()
238238
};
239+
let ctx = if is_validity {
240+
ctx
241+
} else {
242+
ctx.with_field(name.clone())
243+
};
239244
let segment_sink = Arc::clone(&segment_sink);
240245
handle.spawn_nested(move |_| {
241246
// Validity is written through the validity strategy; every other field
@@ -288,8 +293,8 @@ mod tests {
288293
use crate::layouts::flat::writer::FlatLayoutStrategy;
289294
use crate::layouts::table::TableStrategy;
290295
use crate::layouts::zoned::Zoned;
291-
use crate::layouts::zoned::writer::ZonedLayoutOptions;
292-
use crate::layouts::zoned::writer::ZonedStrategy;
296+
use crate::layouts::zoned::writer::ZoneOptions;
297+
use crate::layouts::zoned::writer::ZonedStrategyBuilder;
293298
use crate::segments::TestSegments;
294299
use crate::sequence::SequentialArrayStreamExt;
295300
use crate::test::FailingStrategy;
@@ -300,14 +305,16 @@ mod tests {
300305
fn field_has_zone_map(validity: Validity) -> VortexResult<bool> {
301306
let strategy = StructStrategy::new(
302307
Arc::new(FlatLayoutStrategy::default()),
303-
Arc::new(ZonedStrategy::new(
304-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
305-
FlatLayoutStrategy::default(),
306-
ZonedLayoutOptions {
307-
block_size: NonZeroUsize::new(3).vortex_expect("non zero"),
308-
..Default::default()
309-
},
310-
)),
308+
Arc::new(
309+
ZonedStrategyBuilder::new()
310+
.with_zones([ZoneOptions::new(
311+
NonZeroUsize::new(3).vortex_expect("non zero"),
312+
)])
313+
.build(
314+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
315+
FlatLayoutStrategy::default(),
316+
),
317+
),
311318
);
312319
let is_nullable = validity.nullability().is_nullable();
313320
let (ptr, eof) = SequenceId::root().split();
@@ -340,14 +347,16 @@ mod tests {
340347
fn write_table(array: StructArray) -> VortexResult<LayoutRef> {
341348
let strategy = TableStrategy::new(
342349
Arc::new(FlatLayoutStrategy::default()),
343-
Arc::new(ZonedStrategy::new(
344-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
345-
FlatLayoutStrategy::default(),
346-
ZonedLayoutOptions {
347-
block_size: NonZeroUsize::new(3).vortex_expect("non zero"),
348-
..Default::default()
349-
},
350-
)),
350+
Arc::new(
351+
ZonedStrategyBuilder::new()
352+
.with_zones([ZoneOptions::new(
353+
NonZeroUsize::new(3).vortex_expect("non zero"),
354+
)])
355+
.build(
356+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
357+
FlatLayoutStrategy::default(),
358+
),
359+
),
351360
);
352361
let (ptr, eof) = SequenceId::root().split();
353362
let stream = array.into_array().to_array_stream().sequenced(ptr);

‎vortex-layout/src/layouts/table.rs‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -350,8 +350,8 @@ mod tests {
350350
use crate::layouts::repartition::RepartitionWriterOptions;
351351
use crate::layouts::table::TableStrategy;
352352
use crate::layouts::zoned::Zoned;
353-
use crate::layouts::zoned::writer::ZonedLayoutOptions;
354-
use crate::layouts::zoned::writer::ZonedStrategy;
353+
use crate::layouts::zoned::writer::ZoneOptions;
354+
use crate::layouts::zoned::writer::ZonedStrategyBuilder;
355355
use crate::segments::TestSegments;
356356
use crate::sequence::SequenceId;
357357
use crate::sequence::SequentialArrayStreamExt;
@@ -531,14 +531,9 @@ mod tests {
531531
Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()));
532532
let dispatcher = TableStrategy::new(Arc::clone(&flat), chunked).with_list_layout_factory(
533533
move |list_layout| {
534-
let zoned = ZonedStrategy::new(
535-
list_layout,
536-
Arc::clone(&stats),
537-
ZonedLayoutOptions {
538-
block_size: row_block_size,
539-
..Default::default()
540-
},
541-
);
534+
let zoned = ZonedStrategyBuilder::new()
535+
.with_zones([ZoneOptions::new(row_block_size)])
536+
.build(list_layout, Arc::clone(&stats));
542537
Arc::new(RepartitionStrategy::new(
543538
zoned,
544539
RepartitionWriterOptions {

‎vortex-layout/src/layouts/zoned/reader.rs‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,8 @@ mod test {
286286
use crate::layouts::zoned::ZoneMapSchema;
287287
use crate::layouts::zoned::Zoned;
288288
use crate::layouts::zoned::ZonedLayout;
289-
use crate::layouts::zoned::writer::ZonedLayoutOptions;
290-
use crate::layouts::zoned::writer::ZonedStrategy;
289+
use crate::layouts::zoned::writer::ZoneOptions;
290+
use crate::layouts::zoned::writer::ZonedStrategyBuilder;
291291
use crate::segments::SegmentSource;
292292
use crate::segments::TestSegments;
293293
use crate::sequence::SequenceId;
@@ -309,14 +309,14 @@ mod test {
309309
let ctx = ArrayContext::empty();
310310
let segments = Arc::new(TestSegments::default());
311311
let (ptr, eof) = SequenceId::root().split();
312-
let strategy = ZonedStrategy::new(
313-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
314-
FlatLayoutStrategy::default(),
315-
ZonedLayoutOptions {
316-
block_size: NonZeroUsize::new(3).vortex_expect("non zero"),
317-
..Default::default()
318-
},
319-
);
312+
let strategy = ZonedStrategyBuilder::new()
313+
.with_zones([ZoneOptions::new(
314+
NonZeroUsize::new(3).vortex_expect("non zero"),
315+
)])
316+
.build(
317+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
318+
FlatLayoutStrategy::default(),
319+
);
320320
let array_stream = ChunkedArray::from_iter([
321321
buffer![1, 2, 3].into_array(),
322322
buffer![4, 5, 6].into_array(),
@@ -398,14 +398,14 @@ mod test {
398398
let ctx = ArrayContext::empty();
399399
let segments = Arc::new(TestSegments::default());
400400
let (ptr, eof) = SequenceId::root().split();
401-
let strategy = ZonedStrategy::new(
402-
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
403-
FlatLayoutStrategy::default(),
404-
ZonedLayoutOptions {
405-
block_size: NonZeroUsize::new(3).vortex_expect("non zero"),
406-
..Default::default()
407-
},
408-
);
401+
let strategy = ZonedStrategyBuilder::new()
402+
.with_zones([ZoneOptions::new(
403+
NonZeroUsize::new(3).vortex_expect("non zero"),
404+
)])
405+
.build(
406+
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),
407+
FlatLayoutStrategy::default(),
408+
);
409409
let array_stream = ChunkedArray::from_iter([
410410
PrimitiveArray::new(
411411
buffer![0i32, 0, 0],

0 commit comments

Comments
 (0)