Skip to content

Commit 3edc2c5

Browse files
committed
let the probe compressor canonicalize and take the sample dtype from the first chunk
Signed-off-by: Matt Katz <[email protected]>
1 parent 4986b12 commit 3edc2c5

1 file changed

Lines changed: 15 additions & 13 deletions

File tree

‎vortex-layout/src/layouts/dict/writer.rs‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ use futures::stream::iter;
2121
use futures::stream::once;
2222
use futures::try_join;
2323
use vortex_array::ArrayRef;
24-
use vortex_array::Canonical;
2524
use vortex_array::ExecutionCtx;
2625
use vortex_array::IntoArray;
2726
use vortex_array::VortexSessionExecute;
@@ -165,14 +164,13 @@ impl LayoutStrategy for DictStrategy {
165164
let dtype = stream.dtype().clone();
166165

167166
// 0. decide if chunks are eligible for dict encoding
168-
let (stream, sample) = peek_probe_sample(stream, &dtype, options.probe_len).await?;
167+
let (stream, sample) = peek_probe_sample(stream, options.probe_len).await?;
169168
let stream = SequentialStreamAdapter::new(dtype.clone(), stream).sendable();
170169

171170
let should_fallback = match sample {
172171
None => true, // empty stream
173172
Some(sample) => {
174173
let mut exec_ctx = session.create_execution_ctx();
175-
let sample = sample.execute::<Canonical>(&mut exec_ctx)?.into_array();
176174
let compressed = self
177175
.probe_compressor
178176
.compress_chunk(&sample, &mut exec_ctx)?;
@@ -531,7 +529,6 @@ impl Stream for DictionaryTransformer {
531529
/// front and its first `probe_len` rows, or `None` if the stream is empty.
532530
async fn peek_probe_sample(
533531
mut stream: BoxStream<'static, SequencedChunk>,
534-
dtype: &DType,
535532
probe_len: usize,
536533
) -> VortexResult<(BoxStream<'static, SequencedChunk>, Option<ArrayRef>)> {
537534
let mut peeked = Vec::new();
@@ -544,14 +541,17 @@ async fn peek_probe_sample(
544541
peeked_len += chunk.len();
545542
peeked.push((sequence_id, chunk));
546543
}
547-
if peeked.is_empty() {
548-
return Ok((stream, None));
549-
}
550-
551-
let sample =
552-
ChunkedArray::try_new(peeked.iter().map(|(_, chunk)| chunk.clone()), dtype.clone())?
553-
.into_array()
554-
.slice(0..peeked_len.min(probe_len))?;
544+
let sample_len = peeked_len.min(probe_len);
545+
let sample = match peeked.as_slice() {
546+
[] => return Ok((stream, None)),
547+
[(_, chunk)] => chunk.slice(0..sample_len)?,
548+
[(_, first), ..] => ChunkedArray::try_new(
549+
peeked.iter().map(|(_, chunk)| chunk.clone()),
550+
first.dtype().clone(),
551+
)?
552+
.into_array()
553+
.slice(0..sample_len)?,
554+
};
555555
let stream = iter(peeked.into_iter().map(Ok)).chain(stream).boxed();
556556
Ok((stream, Some(sample)))
557557
}
@@ -609,6 +609,7 @@ mod tests {
609609
use rstest::rstest;
610610
use vortex_array::ArrayContext;
611611
use vortex_array::ArrayRef;
612+
use vortex_array::Canonical;
612613
use vortex_array::ExecutionCtx;
613614
use vortex_array::IntoArray;
614615
use vortex_array::MaskFuture;
@@ -820,7 +821,8 @@ mod tests {
820821
if chunk.len() < PROBE_LEN {
821822
return Ok(chunk.clone());
822823
}
823-
Ok(dict_encode(chunk, ctx)?.into_array())
824+
let chunk = chunk.clone().execute::<Canonical>(ctx)?.into_array();
825+
Ok(dict_encode(&chunk, ctx)?.into_array())
824826
};
825827
let strategy = DictStrategy::new(
826828
ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()),

0 commit comments

Comments
 (0)