@@ -44,6 +44,23 @@ use crate::stats::ArrayAndStats;
4444use crate :: stats:: GenerateStatsOptions ;
4545use crate :: trace;
4646
47+ /// The outcome of [`CascadingCompressor::choose_and_compress`].
48+ enum Selection {
49+ /// A scheme, or the compressor's own empty, null or constant handling, produced a new array.
50+ Compressed ( ArrayRef ) ,
51+ /// Nothing beat the input, which is handed back as it was.
52+ Unchanged ( ArrayRef ) ,
53+ }
54+
55+ impl Selection {
56+ /// Returns the array, compressed or not.
57+ fn into_array ( self ) -> ArrayRef {
58+ match self {
59+ Selection :: Compressed ( array) | Selection :: Unchanged ( array) => array,
60+ }
61+ }
62+ }
63+
4764impl CascadingCompressor {
4865 /// Compresses an array using cascading adaptive compression.
4966 ///
@@ -114,15 +131,15 @@ impl CascadingCompressor {
114131 ) -> VortexResult < ArrayRef > {
115132 match array {
116133 Canonical :: Null ( null_array) => Ok ( null_array. into_array ( ) ) ,
117- Canonical :: Bool ( bool_array) => {
118- self . choose_and_compress ( Canonical :: Bool ( bool_array) , compress_ctx, exec_ctx)
119- }
120- Canonical :: Primitive ( primitive) => {
121- self . choose_and_compress ( Canonical :: Primitive ( primitive) , compress_ctx, exec_ctx)
122- }
123- Canonical :: Decimal ( decimal) => {
124- self . choose_and_compress ( Canonical :: Decimal ( decimal) , compress_ctx, exec_ctx)
125- }
134+ Canonical :: Bool ( bool_array) => self
135+ . choose_and_compress ( Canonical :: Bool ( bool_array) , compress_ctx, exec_ctx)
136+ . map ( Selection :: into_array ) ,
137+ Canonical :: Primitive ( primitive) => self
138+ . choose_and_compress ( Canonical :: Primitive ( primitive) , compress_ctx, exec_ctx)
139+ . map ( Selection :: into_array ) ,
140+ Canonical :: Decimal ( decimal) => self
141+ . choose_and_compress ( Canonical :: Decimal ( decimal) , compress_ctx, exec_ctx)
142+ . map ( Selection :: into_array ) ,
126143 Canonical :: Struct ( struct_array) => {
127144 let fields = struct_array
128145 . iter_unmasked_fields ( )
@@ -169,26 +186,26 @@ impl CascadingCompressor {
169186 ) ?
170187 . into_array ( ) )
171188 }
172- Canonical :: VarBinView ( varbinview) => {
173- self . choose_and_compress ( Canonical :: VarBinView ( varbinview) , compress_ctx, exec_ctx)
174- }
189+ Canonical :: VarBinView ( varbinview) => self
190+ . choose_and_compress ( Canonical :: VarBinView ( varbinview) , compress_ctx, exec_ctx)
191+ . map ( Selection :: into_array ) ,
175192 Canonical :: Extension ( ext_array) => {
176193 // Try scheme-based compression first.
177- let scheme_compressed = self . choose_and_compress (
194+ let selection = self . choose_and_compress (
178195 Canonical :: Extension ( ext_array. clone ( ) ) ,
179196 compress_ctx,
180197 exec_ctx,
181198 ) ?;
182199
183200 // A constant extension array (that might be masked) is already in its terminal
184201 // representation, and compressing the storage separately cannot do better.
185- if scheme_compressed . is :: < Constant > ( ) {
186- return Ok ( scheme_compressed ) ;
187- }
188- if let Some ( masked ) = scheme_compressed . as_opt :: < Masked > ( )
189- && masked. child ( ) . is :: < Constant > ( )
202+ if let Selection :: Compressed ( compressed ) = & selection
203+ && ( compressed . is :: < Constant > ( )
204+ || compressed
205+ . as_opt :: < Masked > ( )
206+ . is_some_and ( | masked| masked . child ( ) . is :: < Constant > ( ) ) )
190207 {
191- return Ok ( scheme_compressed ) ;
208+ return Ok ( selection . into_array ( ) ) ;
192209 }
193210
194211 // Also compress the underlying storage array. Some extension schemes can beat the
@@ -198,10 +215,15 @@ impl CascadingCompressor {
198215 ExtensionArray :: new ( ext_array. ext_dtype ( ) . clone ( ) , compressed_storage)
199216 . into_array ( ) ;
200217
201- if scheme_compressed. nbytes ( ) < storage_compressed. nbytes ( ) {
202- Ok ( scheme_compressed)
203- } else {
204- Ok ( storage_compressed)
218+ match selection {
219+ Selection :: Compressed ( scheme_compressed)
220+ if scheme_compressed. nbytes ( ) < storage_compressed. nbytes ( ) =>
221+ {
222+ Ok ( scheme_compressed)
223+ }
224+ // A canonical extension array's storage is only canonical at its top level,
225+ // so an unchanged input can still hold lazy, unserializable arrays.
226+ Selection :: Compressed ( _) | Selection :: Unchanged ( _) => Ok ( storage_compressed) ,
205227 }
206228 }
207229 Canonical :: Variant ( variant_array) => {
@@ -231,8 +253,10 @@ impl CascadingCompressor {
231253 /// ratio.
232254 ///
233255 /// If a winner is found and its compressed output is actually smaller than the exact
234- /// uncompressed size, that output is returned. Otherwise, the original array is returned
235- /// unchanged.
256+ /// uncompressed size, that output is returned as [`Selection::Compressed`]. Otherwise, the
257+ /// original array is returned as [`Selection::Unchanged`], and the caller decides whether it
258+ /// is a valid result: it is for leaf arrays, but not for arrays like extensions whose children
259+ /// have yet to be compressed.
236260 ///
237261 /// Empty, all-null, and constant arrays are handled by the compressor itself before any
238262 /// scheme evaluation (constant detection is skipped while compressing samples).
@@ -244,7 +268,7 @@ impl CascadingCompressor {
244268 canonical : Canonical ,
245269 compress_ctx : CompressorContext ,
246270 exec_ctx : & mut ExecutionCtx ,
247- ) -> VortexResult < ArrayRef > {
271+ ) -> VortexResult < Selection > {
248272 let eligible_schemes: Vec < & ' static dyn Scheme > = self
249273 . schemes
250274 . iter ( )
@@ -255,13 +279,13 @@ impl CascadingCompressor {
255279 let array: ArrayRef = canonical. into ( ) ;
256280
257281 if array. is_empty ( ) {
258- return Ok ( array) ;
282+ return Ok ( Selection :: Unchanged ( array) ) ;
259283 }
260284
261285 if array. all_invalid ( exec_ctx) ? {
262- return Ok (
286+ return Ok ( Selection :: Compressed (
263287 ConstantArray :: new ( Scalar :: null ( array. dtype ( ) . clone ( ) ) , array. len ( ) ) . into_array ( ) ,
264- ) ;
288+ ) ) ;
265289 }
266290
267291 // Constant detection is built into the compressor: a constant leaf always short-circuits
@@ -270,7 +294,7 @@ impl CascadingCompressor {
270294 //
271295 // Compare valid values directly so even nullable constants avoid distinct-value counting.
272296 if !compress_ctx. is_sample ( ) && constant:: is_constant_for_compression ( & array, exec_ctx) ? {
273- return constant:: compress_as_constant ( array, exec_ctx) ;
297+ return constant:: compress_as_constant ( array, exec_ctx) . map ( Selection :: Compressed ) ;
274298 }
275299
276300 // A scheme competes against the bytes the rows reference, not dead bytes in the buffers.
@@ -286,13 +310,13 @@ impl CascadingCompressor {
286310 let data = ArrayAndStats :: new ( array, merged_opts) ;
287311
288312 if eligible_schemes. is_empty ( ) {
289- return Ok ( data. into_array ( ) ) ;
313+ return Ok ( Selection :: Unchanged ( data. into_array ( ) ) ) ;
290314 }
291315
292316 let Some ( ( winner, winner_estimate) ) =
293317 self . choose_best_scheme ( & eligible_schemes, & data, compress_ctx. clone ( ) , exec_ctx) ?
294318 else {
295- return Ok ( data. into_array ( ) ) ;
319+ return Ok ( Selection :: Unchanged ( data. into_array ( ) ) ) ;
296320 } ;
297321
298322 // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the
@@ -321,9 +345,9 @@ impl CascadingCompressor {
321345 ) ;
322346
323347 if accepted {
324- Ok ( compressed)
348+ Ok ( Selection :: Compressed ( compressed) )
325349 } else {
326- Ok ( data. into_array ( ) )
350+ Ok ( Selection :: Unchanged ( data. into_array ( ) ) )
327351 }
328352 }
329353}
0 commit comments