Skip to content

Commit 6654b2d

Browse files
clauderobert3005
authored andcommitted
Don't pass extension arrays through the compressor unchanged
When no scheme compresses a canonical extension array, `choose_and_compress` returns the input itself, and the extension branch kept it whenever its `nbytes` beat the compressed storage. Canonicalizing an extension array only canonicalizes the top level of its storage, so the input can still hold lazy arrays such as `vortex.slice`, which cannot be serialized. A UUID column mixing a dense chunk with a slice of a sparse one hit this on write (#10417). Use the compressed storage whenever no scheme produced a new array. Signed-off-by: Claude <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_015XzXtnsrnEsBNVa9R2VqKQ
1 parent 8c2534e commit 6654b2d

2 files changed

Lines changed: 90 additions & 0 deletions

File tree

‎vortex-compressor/src/compressor/cascade.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ impl CascadingCompressor {
173173
self.choose_and_compress(Canonical::VarBinView(varbinview), compress_ctx, exec_ctx)
174174
}
175175
Canonical::Extension(ext_array) => {
176+
let input = ext_array.clone().into_array();
177+
176178
// Try scheme-based compression first.
177179
let scheme_compressed = self.choose_and_compress(
178180
Canonical::Extension(ext_array.clone()),
@@ -198,6 +200,13 @@ impl CascadingCompressor {
198200
ExtensionArray::new(ext_array.ext_dtype().clone(), compressed_storage)
199201
.into_array();
200202

203+
// No scheme compressed the array, so `scheme_compressed` is the input itself. A
204+
// canonical extension array's storage can still be lazy (sliced, chunked, ...) and
205+
// need not be serializable, so only the compressed storage is a valid result.
206+
if ArrayRef::ptr_eq(&scheme_compressed, &input) {
207+
return Ok(storage_compressed);
208+
}
209+
201210
if scheme_compressed.nbytes() < storage_compressed.nbytes() {
202211
Ok(scheme_compressed)
203212
} else {

‎vortex-compressor/src/compressor/tests.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,16 @@ use vortex_array::VortexSessionExecute;
1414
use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeOpts;
1515
use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::uncompressed_size_in_bytes;
1616
use vortex_array::arrays::BoolArray;
17+
use vortex_array::arrays::ChunkedArray;
1718
use vortex_array::arrays::Constant;
19+
use vortex_array::arrays::DictArray;
20+
use vortex_array::arrays::ExtensionArray;
21+
use vortex_array::arrays::FixedSizeListArray;
1822
use vortex_array::arrays::Map;
1923
use vortex_array::arrays::NullArray;
2024
use vortex_array::arrays::PrimitiveArray;
25+
use vortex_array::arrays::Slice;
26+
use vortex_array::arrays::SliceArray;
2127
use vortex_array::arrays::VarBinView;
2228
use vortex_array::arrays::VarBinViewArray;
2329
use vortex_array::arrays::varbinview::BinaryView;
@@ -27,12 +33,17 @@ use vortex_array::dtype::DType;
2733
use vortex_array::dtype::MapDType;
2834
use vortex_array::dtype::Nullability;
2935
use vortex_array::dtype::PType;
36+
use vortex_array::dtype::extension::ExtDType;
37+
use vortex_array::dtype::extension::ExtId;
38+
use vortex_array::dtype::extension::ExtVTable;
3039
use vortex_array::scalar::Scalar;
40+
use vortex_array::scalar::ScalarValue;
3141
use vortex_array::validity::Validity;
3242
use vortex_buffer::Buffer;
3343
use vortex_buffer::ByteBuffer;
3444
use vortex_buffer::buffer;
3545
use vortex_error::VortexResult;
46+
use vortex_error::vortex_ensure;
3647
use vortex_session::VortexSession;
3748

3849
use super::CascadingCompressor;
@@ -881,3 +892,73 @@ fn scheme_must_beat_referenced_bytes_not_held_bytes() -> VortexResult<()> {
881892
assert_arrays_eq!(&compressed, &array, &mut exec_ctx);
882893
Ok(())
883894
}
895+
896+
/// An extension type over fixed-size lists, like UUIDs over 16 bytes.
897+
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
898+
struct BytesExtension;
899+
900+
impl ExtVTable for BytesExtension {
901+
type Metadata = u8;
902+
type NativeValue<'a> = &'a ScalarValue;
903+
904+
#[expect(clippy::disallowed_methods, reason = "test-only id")]
905+
fn id(&self) -> ExtId {
906+
ExtId::new("test.bytes")
907+
}
908+
909+
fn serialize_metadata(&self, metadata: &u8) -> VortexResult<Vec<u8>> {
910+
Ok(vec![*metadata])
911+
}
912+
913+
fn deserialize_metadata(&self, metadata: &[u8]) -> VortexResult<u8> {
914+
vortex_ensure!(metadata.len() == 1, "expected one metadata byte");
915+
Ok(metadata[0])
916+
}
917+
918+
fn validate_dtype(dtype: &ExtDType<Self>) -> VortexResult<()> {
919+
vortex_ensure!(
920+
matches!(dtype.storage_dtype(), DType::FixedSizeList(..)),
921+
"expected fixed-size list storage"
922+
);
923+
Ok(())
924+
}
925+
926+
fn unpack_native<'a>(
927+
_dtype: &'a ExtDType<Self>,
928+
value: &'a ScalarValue,
929+
) -> VortexResult<&'a ScalarValue> {
930+
Ok(value)
931+
}
932+
}
933+
934+
/// Canonicalizing an extension array only canonicalizes its storage's top level, so the elements
935+
/// of fixed-size list storage can stay a lazy slice. When no scheme compresses such an array it
936+
/// must not be returned as-is, because the slice is not serializable.
937+
#[test]
938+
fn extension_with_lazy_storage_is_not_passed_through() -> VortexResult<()> {
939+
// Dictionaries of wide values keep the lazy array smaller than its canonical form, and
940+
// chunked arrays only slice on the execute path, so the slice stays lazy.
941+
let dict = |offset: u64| -> VortexResult<ArrayRef> {
942+
let codes = PrimitiveArray::from_iter((0..1024u32).map(|i| (i % 4) as u8)).into_array();
943+
let values = PrimitiveArray::from_iter([offset, offset + 1, offset + 2, offset + 3]);
944+
Ok(DictArray::try_new(codes, values.into_array())?.into_array())
945+
};
946+
let chunks = vec![dict(0)?, dict(100)?];
947+
let elements_dtype = chunks[0].dtype().clone();
948+
let chunked = ChunkedArray::try_new(chunks, elements_dtype)?.into_array();
949+
let elements = SliceArray::new(chunked, 16..2048).into_array();
950+
let storage = FixedSizeListArray::new(elements, 16, Validity::NonNullable, 127).into_array();
951+
let dtype = ExtDType::<BytesExtension>::try_new(0, storage.dtype().clone())?.erased();
952+
let array = ExtensionArray::new(dtype, storage).into_array();
953+
let mut exec_ctx = SESSION.create_execution_ctx();
954+
955+
let compressed = CascadingCompressor::new(Vec::new()).compress(&array, &mut exec_ctx)?;
956+
957+
assert!(
958+
!compressed.depth_first_traversal().any(|a| a.is::<Slice>()),
959+
"compressed array still contains a lazy slice:\n{}",
960+
compressed.display_tree()
961+
);
962+
assert_arrays_eq!(&compressed, &array, &mut exec_ctx);
963+
Ok(())
964+
}

0 commit comments

Comments
 (0)