Skip to content

Commit 8d11832

Browse files
clauderobert3005
authored andcommitted
Make the compressor's unchanged fallback explicit
`choose_and_compress` returned its input whenever no scheme beat it, so every caller got an `ArrayRef` that may or may not have been compressed. That is a valid result for leaf arrays, but not for extension arrays, whose canonical form leaves the storage below its top level lazy; the extension branch had to detect the passthrough by pointer identity. Return a `Selection` that says whether anything was compressed, so each caller picks its fallback explicitly and the extension branch falls back to its compressed storage by construction. Signed-off-by: Claude <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_015XzXtnsrnEsBNVa9R2VqKQ
1 parent 6654b2d commit 8d11832

1 file changed

Lines changed: 58 additions & 43 deletions

File tree

‎vortex-compressor/src/compressor/cascade.rs‎

Lines changed: 58 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,23 @@ use crate::stats::ArrayAndStats;
4444
use crate::stats::GenerateStatsOptions;
4545
use crate::trace;
4646

47+
/// The outcome of [`CascadingCompressor::choose_and_compress`].
48+
enum Selection {
49+
/// A scheme, or the compressor's own empty, null or constant handling, produced a new array.
50+
Compressed(ArrayRef),
51+
/// Nothing beat the input, which is handed back as it was.
52+
Unchanged(ArrayRef),
53+
}
54+
55+
impl Selection {
56+
/// Returns the array, compressed or not.
57+
fn into_array(self) -> ArrayRef {
58+
match self {
59+
Selection::Compressed(array) | Selection::Unchanged(array) => array,
60+
}
61+
}
62+
}
63+
4764
impl CascadingCompressor {
4865
/// Compresses an array using cascading adaptive compression.
4966
///
@@ -114,15 +131,15 @@ impl CascadingCompressor {
114131
) -> VortexResult<ArrayRef> {
115132
match array {
116133
Canonical::Null(null_array) => Ok(null_array.into_array()),
117-
Canonical::Bool(bool_array) => {
118-
self.choose_and_compress(Canonical::Bool(bool_array), compress_ctx, exec_ctx)
119-
}
120-
Canonical::Primitive(primitive) => {
121-
self.choose_and_compress(Canonical::Primitive(primitive), compress_ctx, exec_ctx)
122-
}
123-
Canonical::Decimal(decimal) => {
124-
self.choose_and_compress(Canonical::Decimal(decimal), compress_ctx, exec_ctx)
125-
}
134+
Canonical::Bool(bool_array) => self
135+
.choose_and_compress(Canonical::Bool(bool_array), compress_ctx, exec_ctx)
136+
.map(Selection::into_array),
137+
Canonical::Primitive(primitive) => self
138+
.choose_and_compress(Canonical::Primitive(primitive), compress_ctx, exec_ctx)
139+
.map(Selection::into_array),
140+
Canonical::Decimal(decimal) => self
141+
.choose_and_compress(Canonical::Decimal(decimal), compress_ctx, exec_ctx)
142+
.map(Selection::into_array),
126143
Canonical::Struct(struct_array) => {
127144
let fields = struct_array
128145
.iter_unmasked_fields()
@@ -169,28 +186,26 @@ impl CascadingCompressor {
169186
)?
170187
.into_array())
171188
}
172-
Canonical::VarBinView(varbinview) => {
173-
self.choose_and_compress(Canonical::VarBinView(varbinview), compress_ctx, exec_ctx)
174-
}
189+
Canonical::VarBinView(varbinview) => self
190+
.choose_and_compress(Canonical::VarBinView(varbinview), compress_ctx, exec_ctx)
191+
.map(Selection::into_array),
175192
Canonical::Extension(ext_array) => {
176-
let input = ext_array.clone().into_array();
177-
178193
// Try scheme-based compression first.
179-
let scheme_compressed = self.choose_and_compress(
194+
let selection = self.choose_and_compress(
180195
Canonical::Extension(ext_array.clone()),
181196
compress_ctx,
182197
exec_ctx,
183198
)?;
184199

185200
// A constant extension array (that might be masked) is already in its terminal
186201
// representation, and compressing the storage separately cannot do better.
187-
if scheme_compressed.is::<Constant>() {
188-
return Ok(scheme_compressed);
189-
}
190-
if let Some(masked) = scheme_compressed.as_opt::<Masked>()
191-
&& masked.child().is::<Constant>()
202+
if let Selection::Compressed(compressed) = &selection
203+
&& (compressed.is::<Constant>()
204+
|| compressed
205+
.as_opt::<Masked>()
206+
.is_some_and(|masked| masked.child().is::<Constant>()))
192207
{
193-
return Ok(scheme_compressed);
208+
return Ok(selection.into_array());
194209
}
195210

196211
// Also compress the underlying storage array. Some extension schemes can beat the
@@ -200,17 +215,15 @@ impl CascadingCompressor {
200215
ExtensionArray::new(ext_array.ext_dtype().clone(), compressed_storage)
201216
.into_array();
202217

203-
// No scheme compressed the array, so `scheme_compressed` is the input itself. A
204-
// canonical extension array's storage can still be lazy (sliced, chunked, ...) and
205-
// need not be serializable, so only the compressed storage is a valid result.
206-
if ArrayRef::ptr_eq(&scheme_compressed, &input) {
207-
return Ok(storage_compressed);
208-
}
209-
210-
if scheme_compressed.nbytes() < storage_compressed.nbytes() {
211-
Ok(scheme_compressed)
212-
} else {
213-
Ok(storage_compressed)
218+
match selection {
219+
Selection::Compressed(scheme_compressed)
220+
if scheme_compressed.nbytes() < storage_compressed.nbytes() =>
221+
{
222+
Ok(scheme_compressed)
223+
}
224+
// A canonical extension array's storage is only canonical at its top level,
225+
// so an unchanged input can still hold lazy, unserializable arrays.
226+
Selection::Compressed(_) | Selection::Unchanged(_) => Ok(storage_compressed),
214227
}
215228
}
216229
Canonical::Variant(variant_array) => {
@@ -240,8 +253,10 @@ impl CascadingCompressor {
240253
/// ratio.
241254
///
242255
/// If a winner is found and its compressed output is actually smaller than the exact
243-
/// uncompressed size, that output is returned. Otherwise, the original array is returned
244-
/// unchanged.
256+
/// uncompressed size, that output is returned as [`Selection::Compressed`]. Otherwise, the
257+
/// original array is returned as [`Selection::Unchanged`], and the caller decides whether it
258+
/// is a valid result: it is for leaf arrays, but not for arrays like extensions whose children
259+
/// have yet to be compressed.
245260
///
246261
/// Empty, all-null, and constant arrays are handled by the compressor itself before any
247262
/// scheme evaluation (constant detection is skipped while compressing samples).
@@ -253,7 +268,7 @@ impl CascadingCompressor {
253268
canonical: Canonical,
254269
compress_ctx: CompressorContext,
255270
exec_ctx: &mut ExecutionCtx,
256-
) -> VortexResult<ArrayRef> {
271+
) -> VortexResult<Selection> {
257272
let eligible_schemes: Vec<&'static dyn Scheme> = self
258273
.schemes
259274
.iter()
@@ -264,13 +279,13 @@ impl CascadingCompressor {
264279
let array: ArrayRef = canonical.into();
265280

266281
if array.is_empty() {
267-
return Ok(array);
282+
return Ok(Selection::Unchanged(array));
268283
}
269284

270285
if array.all_invalid(exec_ctx)? {
271-
return Ok(
286+
return Ok(Selection::Compressed(
272287
ConstantArray::new(Scalar::null(array.dtype().clone()), array.len()).into_array(),
273-
);
288+
));
274289
}
275290

276291
// Constant detection is built into the compressor: a constant leaf always short-circuits
@@ -279,7 +294,7 @@ impl CascadingCompressor {
279294
//
280295
// Compare valid values directly so even nullable constants avoid distinct-value counting.
281296
if !compress_ctx.is_sample() && constant::is_constant_for_compression(&array, exec_ctx)? {
282-
return constant::compress_as_constant(array, exec_ctx);
297+
return constant::compress_as_constant(array, exec_ctx).map(Selection::Compressed);
283298
}
284299

285300
// A scheme competes against the bytes the rows reference, not dead bytes in the buffers.
@@ -295,13 +310,13 @@ impl CascadingCompressor {
295310
let data = ArrayAndStats::new(array, merged_opts);
296311

297312
if eligible_schemes.is_empty() {
298-
return Ok(data.into_array());
313+
return Ok(Selection::Unchanged(data.into_array()));
299314
}
300315

301316
let Some((winner, winner_estimate)) =
302317
self.choose_best_scheme(&eligible_schemes, &data, compress_ctx.clone(), exec_ctx)?
303318
else {
304-
return Ok(data.into_array());
319+
return Ok(Selection::Unchanged(data.into_array()));
305320
};
306321

307322
// Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the
@@ -330,9 +345,9 @@ impl CascadingCompressor {
330345
);
331346

332347
if accepted {
333-
Ok(compressed)
348+
Ok(Selection::Compressed(compressed))
334349
} else {
335-
Ok(data.into_array())
350+
Ok(Selection::Unchanged(data.into_array()))
336351
}
337352
}
338353
}

0 commit comments

Comments
 (0)