@@ -44,6 +44,23 @@ use crate::stats::ArrayAndStats;
4444use crate :: stats:: GenerateStatsOptions ;
4545use crate :: trace;
4646
47+ /// The outcome of [`CascadingCompressor::choose_and_compress`].
48+ enum Selection {
49+ /// A scheme, or the compressor's own empty, null or constant handling, produced a new array.
50+ Compressed ( ArrayRef ) ,
51+ /// Nothing beat the input, which is handed back as it was.
52+ Unchanged ( ArrayRef ) ,
53+ }
54+
55+ impl Selection {
56+ /// Returns the array, compressed or not.
57+ fn into_array ( self ) -> ArrayRef {
58+ match self {
59+ Selection :: Compressed ( array) | Selection :: Unchanged ( array) => array,
60+ }
61+ }
62+ }
63+
4764impl CascadingCompressor {
4865 /// Compresses an array using cascading adaptive compression.
4966 ///
@@ -114,15 +131,15 @@ impl CascadingCompressor {
114131 ) -> VortexResult < ArrayRef > {
115132 match array {
116133 Canonical :: Null ( null_array) => Ok ( null_array. into_array ( ) ) ,
117- Canonical :: Bool ( bool_array) => {
118- self . choose_and_compress ( Canonical :: Bool ( bool_array) , compress_ctx, exec_ctx)
119- }
120- Canonical :: Primitive ( primitive) => {
121- self . choose_and_compress ( Canonical :: Primitive ( primitive) , compress_ctx, exec_ctx)
122- }
123- Canonical :: Decimal ( decimal) => {
124- self . choose_and_compress ( Canonical :: Decimal ( decimal) , compress_ctx, exec_ctx)
125- }
134+ Canonical :: Bool ( bool_array) => self
135+ . choose_and_compress ( Canonical :: Bool ( bool_array) , compress_ctx, exec_ctx)
136+ . map ( Selection :: into_array ) ,
137+ Canonical :: Primitive ( primitive) => self
138+ . choose_and_compress ( Canonical :: Primitive ( primitive) , compress_ctx, exec_ctx)
139+ . map ( Selection :: into_array ) ,
140+ Canonical :: Decimal ( decimal) => self
141+ . choose_and_compress ( Canonical :: Decimal ( decimal) , compress_ctx, exec_ctx)
142+ . map ( Selection :: into_array ) ,
126143 Canonical :: Struct ( struct_array) => {
127144 let fields = struct_array
128145 . iter_unmasked_fields ( )
@@ -169,28 +186,26 @@ impl CascadingCompressor {
169186 ) ?
170187 . into_array ( ) )
171188 }
172- Canonical :: VarBinView ( varbinview) => {
173- self . choose_and_compress ( Canonical :: VarBinView ( varbinview) , compress_ctx, exec_ctx)
174- }
189+ Canonical :: VarBinView ( varbinview) => self
190+ . choose_and_compress ( Canonical :: VarBinView ( varbinview) , compress_ctx, exec_ctx)
191+ . map ( Selection :: into_array ) ,
175192 Canonical :: Extension ( ext_array) => {
176- let input = ext_array. clone ( ) . into_array ( ) ;
177-
178193 // Try scheme-based compression first.
179- let scheme_compressed = self . choose_and_compress (
194+ let selection = self . choose_and_compress (
180195 Canonical :: Extension ( ext_array. clone ( ) ) ,
181196 compress_ctx,
182197 exec_ctx,
183198 ) ?;
184199
185200 // A constant extension array (that might be masked) is already in its terminal
186201 // representation, and compressing the storage separately cannot do better.
187- if scheme_compressed . is :: < Constant > ( ) {
188- return Ok ( scheme_compressed ) ;
189- }
190- if let Some ( masked ) = scheme_compressed . as_opt :: < Masked > ( )
191- && masked. child ( ) . is :: < Constant > ( )
202+ if let Selection :: Compressed ( compressed ) = & selection
203+ && ( compressed . is :: < Constant > ( )
204+ || compressed
205+ . as_opt :: < Masked > ( )
206+ . is_some_and ( | masked| masked . child ( ) . is :: < Constant > ( ) ) )
192207 {
193- return Ok ( scheme_compressed ) ;
208+ return Ok ( selection . into_array ( ) ) ;
194209 }
195210
196211 // Also compress the underlying storage array. Some extension schemes can beat the
@@ -200,17 +215,15 @@ impl CascadingCompressor {
200215 ExtensionArray :: new ( ext_array. ext_dtype ( ) . clone ( ) , compressed_storage)
201216 . into_array ( ) ;
202217
203- // No scheme compressed the array, so `scheme_compressed` is the input itself. A
204- // canonical extension array's storage can still be lazy (sliced, chunked, ...) and
205- // need not be serializable, so only the compressed storage is a valid result.
206- if ArrayRef :: ptr_eq ( & scheme_compressed, & input) {
207- return Ok ( storage_compressed) ;
208- }
209-
210- if scheme_compressed. nbytes ( ) < storage_compressed. nbytes ( ) {
211- Ok ( scheme_compressed)
212- } else {
213- Ok ( storage_compressed)
218+ match selection {
219+ Selection :: Compressed ( scheme_compressed)
220+ if scheme_compressed. nbytes ( ) < storage_compressed. nbytes ( ) =>
221+ {
222+ Ok ( scheme_compressed)
223+ }
224+ // A canonical extension array's storage is only canonical at its top level,
225+ // so an unchanged input can still hold lazy, unserializable arrays.
226+ Selection :: Compressed ( _) | Selection :: Unchanged ( _) => Ok ( storage_compressed) ,
214227 }
215228 }
216229 Canonical :: Variant ( variant_array) => {
@@ -240,8 +253,10 @@ impl CascadingCompressor {
240253 /// ratio.
241254 ///
242255 /// If a winner is found and its compressed output is actually smaller than the exact
243- /// uncompressed size, that output is returned. Otherwise, the original array is returned
244- /// unchanged.
256+ /// uncompressed size, that output is returned as [`Selection::Compressed`]. Otherwise, the
257+ /// original array is returned as [`Selection::Unchanged`], and the caller decides whether it
258+ /// is a valid result: it is for leaf arrays, but not for arrays like extensions whose children
259+ /// have yet to be compressed.
245260 ///
246261 /// Empty, all-null, and constant arrays are handled by the compressor itself before any
247262 /// scheme evaluation (constant detection is skipped while compressing samples).
@@ -253,7 +268,7 @@ impl CascadingCompressor {
253268 canonical : Canonical ,
254269 compress_ctx : CompressorContext ,
255270 exec_ctx : & mut ExecutionCtx ,
256- ) -> VortexResult < ArrayRef > {
271+ ) -> VortexResult < Selection > {
257272 let eligible_schemes: Vec < & ' static dyn Scheme > = self
258273 . schemes
259274 . iter ( )
@@ -264,13 +279,13 @@ impl CascadingCompressor {
264279 let array: ArrayRef = canonical. into ( ) ;
265280
266281 if array. is_empty ( ) {
267- return Ok ( array) ;
282+ return Ok ( Selection :: Unchanged ( array) ) ;
268283 }
269284
270285 if array. all_invalid ( exec_ctx) ? {
271- return Ok (
286+ return Ok ( Selection :: Compressed (
272287 ConstantArray :: new ( Scalar :: null ( array. dtype ( ) . clone ( ) ) , array. len ( ) ) . into_array ( ) ,
273- ) ;
288+ ) ) ;
274289 }
275290
276291 // Constant detection is built into the compressor: a constant leaf always short-circuits
@@ -279,7 +294,7 @@ impl CascadingCompressor {
279294 //
280295 // Compare valid values directly so even nullable constants avoid distinct-value counting.
281296 if !compress_ctx. is_sample ( ) && constant:: is_constant_for_compression ( & array, exec_ctx) ? {
282- return constant:: compress_as_constant ( array, exec_ctx) ;
297+ return constant:: compress_as_constant ( array, exec_ctx) . map ( Selection :: Compressed ) ;
283298 }
284299
285300 // A scheme competes against the bytes the rows reference, not dead bytes in the buffers.
@@ -295,13 +310,13 @@ impl CascadingCompressor {
295310 let data = ArrayAndStats :: new ( array, merged_opts) ;
296311
297312 if eligible_schemes. is_empty ( ) {
298- return Ok ( data. into_array ( ) ) ;
313+ return Ok ( Selection :: Unchanged ( data. into_array ( ) ) ) ;
299314 }
300315
301316 let Some ( ( winner, winner_estimate) ) =
302317 self . choose_best_scheme ( & eligible_schemes, & data, compress_ctx. clone ( ) , exec_ctx) ?
303318 else {
304- return Ok ( data. into_array ( ) ) ;
319+ return Ok ( Selection :: Unchanged ( data. into_array ( ) ) ) ;
305320 } ;
306321
307322 // Run the winning scheme's `compress`. On failure, emit an ERROR event carrying the
@@ -330,9 +345,9 @@ impl CascadingCompressor {
330345 ) ;
331346
332347 if accepted {
333- Ok ( compressed)
348+ Ok ( Selection :: Compressed ( compressed) )
334349 } else {
335- Ok ( data. into_array ( ) )
350+ Ok ( Selection :: Unchanged ( data. into_array ( ) ) )
336351 }
337352 }
338353}
0 commit comments