Skip to content

Commit a997c3f

Browse files
mprammerclaude
andauthored
test(compat): add a map.vortex backward-compatibility fixture (#9463)
## Rationale for this change `vortex.map` joined the frozen `core2026.08.0` edition at Vortex 0.84.0, which makes its layout a backward-compatibility promise, but the compat fixture suite has no map file — nothing pins how a map is written, so the weekly `compat-validation` run has nothing to check and a future reader change could silently break every map file written today. The gap isn't map-specific: `vortex.masked` (frozen since 0.54.0) and `vortex.variant` (frozen since 0.65.0) also have no fixtures, and I'll send those separately. ## What changes are included in this PR? One new fixture, `map.vortex`, in the synthetic array set. It carries a `map<i32, utf8?>` with `keys_sorted = false` including an empty row and a row that repeats a key, since duplicate keys are representable and have to survive in order rather than being deduplicated; a `map<utf8, i64>` with `keys_sorted = true`; and a nullable `map<i32, utf8?>` that keeps a null map distinct from an empty one and holds a present key with a null value. The existing `roundtrip_fixtures_to_bytes` test picks the fixture up automatically, and `vortex-compat generate` followed by `check --mode exact` passes on all 36 fixtures. Worth flagging for review that `build()` is immutable once published, so this content can be added to later but not amended — the cases are the part to scrutinise. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: mprammer <[email protected]> Signed-off-by: Robert Kruszewski <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent 0d9d5b8 commit a997c3f

3 files changed

Lines changed: 180 additions & 3 deletions

File tree

‎vortex-test/compat-gen/src/adapter.rs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ use vortex::VortexSessionDefault;
1515
use vortex::array::ArrayRef;
1616
use vortex::array::MaskFuture;
1717
use vortex::array::expr::root;
18+
use vortex::editions::CORE_2026_08_2;
19+
use vortex::editions::EditionSessionExt;
1820
use vortex::file::OpenOptionsSessionExt;
1921
use vortex::file::WriteOptionsSessionExt;
2022
use vortex::io::session::RuntimeSessionExt;
@@ -34,6 +36,18 @@ fn runtime() -> VortexResult<Runtime> {
3436
Runtime::new().map_err(|e| vortex_err!("failed to create tokio runtime: {e}"))
3537
}
3638

39+
/// Session used to write fixtures.
40+
///
41+
/// Enables the draft `core2026.08.2` edition on top of the defaults so fixtures may pin
42+
/// `vortex.map` before it is frozen into a core edition.
43+
fn fixture_session() -> VortexResult<VortexSession> {
44+
let session = VortexSession::default();
45+
session
46+
.enable_edition(CORE_2026_08_2)
47+
.map_err(|e| vortex_err!("{e}"))?;
48+
Ok(session)
49+
}
50+
3751
/// Compute all statistics on every node in the array tree.
3852
///
3953
/// The flat layout writer does not compute stats itself — it only serializes stats already
@@ -69,9 +83,10 @@ pub fn write_compressed(
6983
strategy: Arc<dyn LayoutStrategy>,
7084
) -> VortexResult<()> {
7185
let stream = ArrayStreamAdapter::new(chunk.dtype().clone(), stream::iter([Ok(chunk)]));
86+
let session = fixture_session()?;
7287

73-
runtime()?.block_on(async {
74-
let session = VortexSession::default().with_tokio();
88+
runtime()?.block_on(async move {
89+
let session = session.with_tokio();
7590
let mut file = tokio::fs::File::create(path)
7691
.await
7792
.map_err(|e| vortex_err!("failed to create {}: {e}", path.display()))?;
@@ -90,7 +105,7 @@ pub fn write_compressed_to_bytes(
90105
chunk: ArrayRef,
91106
strategy: Arc<dyn LayoutStrategy>,
92107
) -> VortexResult<ByteBuffer> {
93-
write_compressed_to_bytes_with_session(&VortexSession::default(), chunk, strategy)
108+
write_compressed_to_bytes_with_session(&fixture_session()?, chunk, strategy)
94109
}
95110

96111
/// Write a `.vortex` file into memory using a caller-provided session and layout strategy,
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use vortex_array::ArrayId;
5+
use vortex_array::ArrayRef;
6+
use vortex_array::ArrayVTable;
7+
use vortex_array::ExecutionCtx;
8+
use vortex_array::IntoArray;
9+
use vortex_array::arrays::ListViewArray;
10+
use vortex_array::arrays::Map;
11+
use vortex_array::arrays::MapArray;
12+
use vortex_array::arrays::PrimitiveArray;
13+
use vortex_array::arrays::StructArray;
14+
use vortex_array::arrays::VarBinArray;
15+
use vortex_array::dtype::DType;
16+
use vortex_array::dtype::FieldNames;
17+
use vortex_array::dtype::MapDType;
18+
use vortex_array::dtype::Nullability;
19+
use vortex_array::dtype::PType;
20+
use vortex_array::validity::Validity;
21+
use vortex_buffer::buffer;
22+
use vortex_error::VortexResult;
23+
24+
use crate::fixtures::FlatLayoutFixture;
25+
26+
pub struct MapFixture;
27+
28+
impl FlatLayoutFixture for MapFixture {
29+
fn name(&self) -> &str {
30+
"map.vortex"
31+
}
32+
33+
fn description(&self) -> &str {
34+
"Map arrays with unsorted and sorted keys, nullable values, and null maps"
35+
}
36+
37+
fn expected_encodings(&self) -> Vec<ArrayId> {
38+
vec![Map.id()]
39+
}
40+
41+
fn build(&self, _ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
42+
// map<i32, utf8?> with keys_sorted = false:
43+
// [{1: "one", 2: null}, {}, {1: "dup-old", 1: "dup-new"}, {5: "five"}]
44+
//
45+
// Row 2 repeats key 1 on purpose: duplicate keys are representable and a reader must
46+
// preserve both entries in order rather than deduplicating them.
47+
let attrs_keys =
48+
PrimitiveArray::new(buffer![1i32, 2, 1, 1, 5], Validity::NonNullable).into_array();
49+
let attrs_values = VarBinArray::from_iter(
50+
[
51+
Some("one"),
52+
None,
53+
Some("dup-old"),
54+
Some("dup-new"),
55+
Some("five"),
56+
],
57+
DType::Utf8(Nullability::Nullable),
58+
)
59+
.into_array();
60+
let attrs_entries = StructArray::try_new(
61+
FieldNames::from(["key", "value"]),
62+
vec![attrs_keys, attrs_values],
63+
5,
64+
Validity::NonNullable,
65+
)?;
66+
let attrs = MapArray::try_new(
67+
MapDType::try_new(
68+
DType::Primitive(PType::I32, Nullability::NonNullable),
69+
DType::Utf8(Nullability::Nullable),
70+
false,
71+
)?,
72+
ListViewArray::try_new(
73+
attrs_entries.into_array(),
74+
PrimitiveArray::new(buffer![0u32, 2, 2, 4], Validity::NonNullable).into_array(),
75+
PrimitiveArray::new(buffer![2u32, 0, 2, 1], Validity::NonNullable).into_array(),
76+
Validity::NonNullable,
77+
)?,
78+
)?;
79+
80+
// map<utf8, i64> with keys_sorted = true:
81+
// [{"a": 1, "b": 2}, {"z": 26}, {}, {"k": 11, "m": 13, "n": 14}]
82+
//
83+
// Keys are sorted within each row, matching the dtype's sortedness assertion.
84+
let sorted_keys = VarBinArray::from_iter(
85+
[
86+
Some("a"),
87+
Some("b"),
88+
Some("z"),
89+
Some("k"),
90+
Some("m"),
91+
Some("n"),
92+
],
93+
DType::Utf8(Nullability::NonNullable),
94+
)
95+
.into_array();
96+
let sorted_values =
97+
PrimitiveArray::new(buffer![1i64, 2, 26, 11, 13, 14], Validity::NonNullable)
98+
.into_array();
99+
let sorted_entries = StructArray::try_new(
100+
FieldNames::from(["key", "value"]),
101+
vec![sorted_keys, sorted_values],
102+
6,
103+
Validity::NonNullable,
104+
)?;
105+
let sorted_attrs = MapArray::try_new(
106+
MapDType::try_new(
107+
DType::Utf8(Nullability::NonNullable),
108+
DType::Primitive(PType::I64, Nullability::NonNullable),
109+
true,
110+
)?,
111+
ListViewArray::try_new(
112+
sorted_entries.into_array(),
113+
PrimitiveArray::new(buffer![0u32, 2, 3, 3], Validity::NonNullable).into_array(),
114+
PrimitiveArray::new(buffer![2u32, 1, 0, 3], Validity::NonNullable).into_array(),
115+
Validity::NonNullable,
116+
)?,
117+
)?;
118+
119+
// Nullable map<i32, utf8?>: [null, {}, {7: "seven"}, {8: null}]
120+
//
121+
// A null map and an empty map are distinct values, and the last row carries a present
122+
// key with a null value.
123+
let nullable_keys =
124+
PrimitiveArray::new(buffer![7i32, 8], Validity::NonNullable).into_array();
125+
let nullable_values =
126+
VarBinArray::from_iter([Some("seven"), None], DType::Utf8(Nullability::Nullable))
127+
.into_array();
128+
let nullable_entries = StructArray::try_new(
129+
FieldNames::from(["key", "value"]),
130+
vec![nullable_keys, nullable_values],
131+
2,
132+
Validity::NonNullable,
133+
)?;
134+
let nullable_attrs = MapArray::try_new(
135+
MapDType::try_new(
136+
DType::Primitive(PType::I32, Nullability::NonNullable),
137+
DType::Utf8(Nullability::Nullable),
138+
false,
139+
)?,
140+
ListViewArray::try_new(
141+
nullable_entries.into_array(),
142+
PrimitiveArray::new(buffer![0u32, 0, 0, 1], Validity::NonNullable).into_array(),
143+
PrimitiveArray::new(buffer![0u32, 0, 1, 1], Validity::NonNullable).into_array(),
144+
Validity::from_iter([false, true, true, true]),
145+
)?,
146+
)?;
147+
148+
let arr = StructArray::try_new(
149+
FieldNames::from(["attrs", "sorted_attrs", "nullable_attrs"]),
150+
vec![
151+
attrs.into_array(),
152+
sorted_attrs.into_array(),
153+
nullable_attrs.into_array(),
154+
],
155+
4,
156+
Validity::NonNullable,
157+
)?;
158+
Ok(arr.into_array())
159+
}
160+
}

‎vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ mod decimal;
1212
mod fixed_size_list;
1313
mod list;
1414
mod listview;
15+
mod map;
1516
mod null;
1617
mod primitive;
1718
mod struct_nested;
@@ -35,5 +36,6 @@ pub fn fixtures() -> Vec<Box<dyn FlatLayoutFixture>> {
3536
Box::new(datetime::DateTimeFixture),
3637
Box::new(decimal::DecimalFixture),
3738
Box::new(listview::ListViewFixture),
39+
Box::new(map::MapFixture),
3840
]
3941
}

0 commit comments

Comments
 (0)