Skip to content

Commit bc9aa10

Browse files
authored
Add Map logical type and scalar support (#9107)
## Rationale for this change Vortex currently has no native logical map type, so Arrow maps must be represented as lists of structs and lose map-specific semantics such as non-null keys and the `keys_sorted` assertion. This provides the type-level foundation for native Map support; #9108 builds on it with canonical arrays and builders. Related to #428. ## What changes are included in this PR? - Adds `DType::Map` and `MapDType`, including key/value dtypes, outer nullability, sortedness, validation, coercion, display, equality, and hashing. - Adds map scalar construction and typed access through `Scalar::map`, `Scalar::try_map`, and `MapScalar`. - Adds map dtype and scalar serialization through Serde, Protobuf, and FlatBuffers. - Adds Arrow dtype/schema round-tripping, Vortex-to-Arrow map scalar conversion, and DataFusion physical-schema reconciliation. - Tests cover dtype and scalar validation, null and empty maps, coercion, serialization round-trips, Arrow schema conversion, and nested extension metadata. ## What APIs are changed? Are there any user-facing changes? This adds the public `DType::Map` and `ScalarValue::Map` variants, `MapDType`, `MapScalar`, and their constructors and accessors. Consumers that exhaustively match `DType` or `ScalarValue` must handle the new variants. The Protobuf and FlatBuffers dtype schemas also gain Map variants. This PR supports Map dtypes, schemas, and scalars, but not full Arrow `MapArray` transport. Signed-off-by: Adam Gutglick <[email protected]>
1 parent fd43ebd commit bc9aa10

59 files changed

Lines changed: 1813 additions & 25 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎encodings/sparse/src/canonical.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ pub(super) fn execute_sparse(parts: SparseParts, ctx: &mut ExecutionCtx) -> Vort
153153
DType::FixedSizeList(.., nullability) => {
154154
execute_sparse_fixed_size_list(&patches, &fill_value, len, *nullability, ctx)?
155155
}
156+
DType::Map(..) => vortex_bail!("Sparse canonicalization does not support Map arrays yet"),
156157
DType::Struct(struct_fields, ..) => execute_sparse_struct(
157158
struct_fields,
158159
fill_value.as_struct(),

‎fuzz/src/array/compare.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,11 @@ pub fn compare_canonical_array(
186186
}))
187187
.into_array()
188188
}
189-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
189+
d @ (DType::Null
190+
| DType::Map(..)
191+
| DType::Union(..)
192+
| DType::Variant(_)
193+
| DType::Extension(_)) => {
190194
unreachable!("DType {d} not supported for fuzzing")
191195
}
192196
}

‎fuzz/src/array/filter.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,11 @@ pub fn filter_canonical_array(
121121
)
122122
.map(|a| a.into_array())
123123
}
124-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
124+
d @ (DType::Null
125+
| DType::Map(..)
126+
| DType::Union(..)
127+
| DType::Variant(_)
128+
| DType::Extension(_)) => {
125129
unreachable!("DType {d} not supported for fuzzing")
126130
}
127131
}

‎fuzz/src/array/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,7 @@ fn actions_for_dtype(dtype: &DType) -> HashSet<ActionType> {
518518
acc.intersection(&actions).copied().collect()
519519
})
520520
}
521+
DType::Map(..) => HashSet::new(),
521522
DType::Union(..) => todo!("TODO(connor)[Union]: unimplemented"),
522523
// Currently, no support at all
523524
DType::Variant(_) => unreachable!("Variant dtype shouldn't be fuzzed"),

‎fuzz/src/array/search_sorted.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,11 @@ pub fn search_sorted_canonical_array(
149149
.collect::<VortexResult<Vec<_>>>()?;
150150
scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side)
151151
}
152-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
152+
d @ (DType::Null
153+
| DType::Map(..)
154+
| DType::Union(..)
155+
| DType::Variant(_)
156+
| DType::Extension(_)) => {
153157
unreachable!("DType {d} not supported for fuzzing")
154158
}
155159
}

‎fuzz/src/array/slice.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,11 @@ pub fn slice_canonical_array(
125125
)
126126
.map(|a| a.into_array())
127127
}
128-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
128+
d @ (DType::Null
129+
| DType::Map(..)
130+
| DType::Union(..)
131+
| DType::Variant(_)
132+
| DType::Extension(_)) => {
129133
unreachable!("DType {d} not supported for fuzzing")
130134
}
131135
}

‎fuzz/src/array/sort.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,11 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR
102102
});
103103
take_canonical_array_non_nullable_indices(array, &sort_indices, ctx)
104104
}
105-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
105+
d @ (DType::Null
106+
| DType::Map(..)
107+
| DType::Union(..)
108+
| DType::Variant(_)
109+
| DType::Extension(_)) => {
106110
unreachable!("DType {d} not supported for fuzzing")
107111
}
108112
}

‎fuzz/src/array/take.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,11 @@ pub fn take_canonical_array(
148148
)
149149
.map(|a| a.into_array())
150150
}
151-
d @ (DType::Null | DType::Union(..) | DType::Variant(_) | DType::Extension(_)) => {
151+
d @ (DType::Null
152+
| DType::Map(..)
153+
| DType::Union(..)
154+
| DType::Variant(_)
155+
| DType::Extension(_)) => {
152156
unreachable!("DType {d} not supported for fuzzing")
153157
}
154158
}

‎vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ impl AggregateFnVTable for IsSorted {
246246
DType::Null
247247
| DType::List(..)
248248
| DType::FixedSizeList(..)
249+
| DType::Map(..)
249250
| DType::Struct(..)
250251
| DType::Union(..)
251252
| DType::Variant(..)
@@ -263,6 +264,7 @@ impl AggregateFnVTable for IsSorted {
263264
DType::Null
264265
| DType::List(..)
265266
| DType::FixedSizeList(..)
267+
| DType::Map(..)
266268
| DType::Struct(..)
267269
| DType::Union(..)
268270
| DType::Variant(..)

‎vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,9 @@ pub(crate) fn constant_uncompressed_size_in_bytes(
236236
let canonical = array.array().clone().execute::<Canonical>(ctx)?;
237237
return canonical_uncompressed_size_in_bytes(&canonical, ctx);
238238
}
239+
DType::Map(..) => {
240+
vortex_bail!("UncompressedSizeInBytes is not supported for map arrays yet")
241+
}
239242
DType::Union(..) => {
240243
todo!(
241244
"TODO(connor)[Union]: support constant Union size accounting after constant Union \
@@ -294,6 +297,7 @@ fn supports_uncompressed_size_in_bytes(dtype: &DType) -> bool {
294297
DType::List(element_dtype, _) | DType::FixedSizeList(element_dtype, ..) => {
295298
supports_uncompressed_size_in_bytes(element_dtype)
296299
}
300+
DType::Map(..) => false,
297301
DType::Struct(fields, _) => fields
298302
.fields()
299303
.all(|field| supports_uncompressed_size_in_bytes(&field)),

0 commit comments

Comments
 (0)