Skip to content

Commit 956c1ea

Browse files
authored
Reject nullable nested levels in list-based spatial types (#11183)
## Summary - Tracking Issue: #11114 GeoArrow only allows nulls at the outer level of a geometry, but the validators of the list-based spatial types only checked the coordinate fields, so `LineString` also accepted `list({x=f64, y=f64}?)`. `LineString`, `MultiPoint`, `Polygon`, `MultiLineString`, and `MultiPolygon` now accept only the storage that their helper builds, at either top-level nullability. That leaves each type one storage form for each dimension, which the storage declarations (SD in #11114) need. ## Changes Each validator compares the stored dtype with its storage helper, so the helpers stay the only definition of the form. `Point` and `Rect` already rejected nullable coordinate fields, and WKB has no nested level. Tests cover every nested level of each type, and reading a serialized `vortex.st.linestring` whose coordinate `Struct` is nullable. ## API Changes Building one of these five types with a nullable inner `List` or coordinate `Struct` (e.g., with `ExtDType::try_new`) now returns an error, and so does reading a dtype in that form. No Vortex writer produces that form, so only storage built by hand through the Rust API can hit it. Note that a file keeps its columns as fields of a struct whose field dtypes are parsed lazily, so reading such a column from a file currently panics instead of returning the error. A follow-up fixes that. Signed-off-by: Connor Tsui <[email protected]>
1 parent d5271f9 commit 956c1ea

6 files changed

Lines changed: 121 additions & 10 deletions

File tree

‎vortex-spatial/src/extension/linestring.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,21 @@ pub(crate) fn linestring_storage_dtype(dim: Dimension, nullability: Nullability)
104104
DType::List(Arc::new(coords), nullability)
105105
}
106106

107-
/// Validate `dtype` is `List<coordinate-struct>` and return its [`Dimension`].
107+
/// Validate that `dtype` equals [`linestring_storage_dtype`] for its [`Dimension`] and top-level
108+
/// nullability, and return that dimension. As in GeoArrow, only the outer `List` can be nullable.
108109
pub(crate) fn linestring_dimension(dtype: &DType) -> VortexResult<Dimension> {
109110
let DType::List(coords, _) = dtype else {
110111
vortex_bail!("linestring storage must be a List of coordinates, was {dtype}");
111112
};
112-
coordinate_dimension(coords)
113+
let dimension = coordinate_dimension(coords)?;
114+
115+
vortex_ensure_eq!(
116+
*dtype,
117+
linestring_storage_dtype(dimension, dtype.nullability()),
118+
"linestring storage can be nullable only at its outer List"
119+
);
120+
121+
Ok(dimension)
113122
}
114123

115124
/// Return one coordinate ordinate, filling an ordinate absent from the point dimension with zero.

‎vortex-spatial/src/extension/mod.rs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,21 +347,32 @@ pub(crate) fn spatial_metadata_from_arrow(metadata: &Metadata) -> SpatialMetadat
347347

348348
#[cfg(test)]
349349
mod tests {
350+
use std::sync::Arc;
350351
use std::sync::LazyLock;
351352

352353
use prost::Message;
354+
use rstest::rstest;
353355
use vortex_array::VortexSessionExecute;
354356
use vortex_array::dtype::DType;
357+
use vortex_array::dtype::Nullability;
358+
use vortex_array::dtype::extension::ExtDType;
359+
use vortex_array::dtype::extension::ExtVTable;
360+
use vortex_array::dtype::proto::dtype as pb;
361+
use vortex_array::dtype::proto::dtype::d_type::DtypeType;
355362
use vortex_error::VortexResult;
356363
use vortex_error::vortex_err;
357364
use vortex_session::VortexSession;
358365

359366
use super::LineString;
360367
use super::MultiLineString;
361368
use super::MultiPoint;
369+
use super::MultiPolygon;
362370
use super::Point;
363371
use super::Polygon;
364372
use crate::extension::SpatialMetadata;
373+
use crate::extension::coordinate::Dimension;
374+
use crate::extension::coordinate::coordinate_storage_dtype;
375+
use crate::test_harness::spatial_session;
365376

366377
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
367378

@@ -546,4 +557,54 @@ mod tests {
546557
assert_eq!(actual_storage, expected_storage);
547558
Ok(())
548559
}
560+
561+
/// A non-nullable `List` of `element`.
562+
fn list(element: DType) -> DType {
563+
DType::List(Arc::new(element), Nullability::NonNullable)
564+
}
565+
566+
/// The non-nullable 2D coordinate `Struct`.
567+
fn xy() -> DType {
568+
coordinate_storage_dtype(Dimension::Xy, Nullability::NonNullable)
569+
}
570+
571+
/// The list-based geometry types reject storage with a nullable inner `List` or coordinate
572+
/// `Struct`, and the error reports the rejected storage.
573+
#[rstest]
574+
#[case::linestring_coordinates( LineString, list(xy().as_nullable()))]
575+
#[case::multipoint_coordinates( MultiPoint, list(xy().as_nullable()))]
576+
#[case::polygon_rings( Polygon, list(list(xy()).as_nullable()))]
577+
#[case::polygon_coordinates( Polygon, list(list(xy().as_nullable())))]
578+
#[case::multilinestring_lines( MultiLineString, list(list(xy()).as_nullable()))]
579+
#[case::multilinestring_coordinates(MultiLineString, list(list(xy().as_nullable())))]
580+
#[case::multipolygon_polygons( MultiPolygon, list(list(list(xy())).as_nullable()))]
581+
#[case::multipolygon_rings( MultiPolygon, list(list(list(xy()).as_nullable())))]
582+
#[case::multipolygon_coordinates( MultiPolygon, list(list(list(xy().as_nullable()))))]
583+
fn nullable_nested_level_is_rejected(
584+
#[case] vtable: impl ExtVTable<Metadata = SpatialMetadata>,
585+
#[case] storage: DType,
586+
) {
587+
let result = ExtDType::try_with_vtable(vtable, SpatialMetadata::default(), storage.clone());
588+
assert!(result.is_err_and(|error| error.to_string().contains(&format!("left: {storage}"))));
589+
}
590+
591+
/// A serialized `vortex.st.linestring` dtype with a nullable coordinate `Struct` fails to read,
592+
/// with an error that names the expected storage. [`LineString`] rejects that storage at
593+
/// construction, so the test builds the serialized extension dtype by hand.
594+
#[test]
595+
fn nullable_nested_level_fails_to_read() -> VortexResult<()> {
596+
let serialized = pb::DType {
597+
dtype_type: Some(DtypeType::Extension(Box::new(pb::Extension {
598+
id: "vortex.st.linestring".to_string(),
599+
storage_dtype: Some(Box::new(pb::DType::try_from(&list(xy().as_nullable()))?)),
600+
metadata: None,
601+
}))),
602+
};
603+
604+
let result = DType::from_proto(&serialized, &spatial_session());
605+
assert!(result.is_err_and(|error| error.to_string().contains(
606+
"linestring storage can be nullable only at its outer List\n left: list({x=f64, y=f64}?)\n right: list({x=f64, y=f64})"
607+
)));
608+
Ok(())
609+
}
549610
}

‎vortex-spatial/src/extension/multilinestring.rs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ use vortex_error::VortexError;
4242
use vortex_error::VortexResult;
4343
use vortex_error::vortex_bail;
4444
use vortex_error::vortex_ensure;
45+
use vortex_error::vortex_ensure_eq;
4546
use vortex_error::vortex_err;
4647
use vortex_session::registry::CachedId;
4748
use vortex_session::registry::Id;
@@ -97,15 +98,25 @@ pub(crate) fn multilinestring_storage_dtype(dim: Dimension, nullability: Nullabi
9798
DType::List(Arc::new(line), nullability)
9899
}
99100

100-
/// Validate `dtype` is `List<List<coordinate-struct>>` and return its [`Dimension`].
101+
/// Validate that `dtype` equals [`multilinestring_storage_dtype`] for its [`Dimension`] and
102+
/// top-level nullability, and return that dimension. As in GeoArrow, only the outer `List` can be
103+
/// nullable.
101104
pub(crate) fn multilinestring_dimension(dtype: &DType) -> VortexResult<Dimension> {
102105
let DType::List(line, _) = dtype else {
103106
vortex_bail!("multilinestring storage must be a List of line strings, was {dtype}");
104107
};
105108
let DType::List(coords, _) = line.as_ref() else {
106109
vortex_bail!("multilinestring line storage must be a List of coordinates, was {line}");
107110
};
108-
coordinate_dimension(coords)
111+
let dimension = coordinate_dimension(coords)?;
112+
113+
vortex_ensure_eq!(
114+
*dtype,
115+
multilinestring_storage_dtype(dimension, dtype.nullability()),
116+
"multilinestring storage can be nullable only at its outer List"
117+
);
118+
119+
Ok(dimension)
109120
}
110121

111122
static ARROW_MULTILINESTRING: CachedId = CachedId::new(MultiLineStringType::NAME);

‎vortex-spatial/src/extension/multipoint.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ use vortex_error::VortexError;
4242
use vortex_error::VortexResult;
4343
use vortex_error::vortex_bail;
4444
use vortex_error::vortex_ensure;
45+
use vortex_error::vortex_ensure_eq;
4546
use vortex_error::vortex_err;
4647
use vortex_session::registry::CachedId;
4748
use vortex_session::registry::Id;
@@ -94,12 +95,21 @@ pub(crate) fn multipoint_storage_dtype(dim: Dimension, nullability: Nullability)
9495
DType::List(Arc::new(coords), nullability)
9596
}
9697

97-
/// Validate `dtype` is `List<coordinate-struct>` and return its [`Dimension`].
98+
/// Validate that `dtype` equals [`multipoint_storage_dtype`] for its [`Dimension`] and top-level
99+
/// nullability, and return that dimension. As in GeoArrow, only the outer `List` can be nullable.
98100
pub(crate) fn multipoint_dimension(dtype: &DType) -> VortexResult<Dimension> {
99101
let DType::List(coords, _) = dtype else {
100102
vortex_bail!("multipoint storage must be a List of coordinates, was {dtype}");
101103
};
102-
coordinate_dimension(coords)
104+
let dimension = coordinate_dimension(coords)?;
105+
106+
vortex_ensure_eq!(
107+
*dtype,
108+
multipoint_storage_dtype(dimension, dtype.nullability()),
109+
"multipoint storage can be nullable only at its outer List"
110+
);
111+
112+
Ok(dimension)
103113
}
104114

105115
static ARROW_MULTIPOINT: CachedId = CachedId::new(MultiPointType::NAME);

‎vortex-spatial/src/extension/multipolygon.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ use vortex_error::VortexError;
4141
use vortex_error::VortexResult;
4242
use vortex_error::vortex_bail;
4343
use vortex_error::vortex_ensure;
44+
use vortex_error::vortex_ensure_eq;
4445
use vortex_error::vortex_err;
4546
use vortex_session::registry::CachedId;
4647
use vortex_session::registry::Id;
@@ -95,7 +96,8 @@ pub(crate) fn multipolygon_storage_dtype(dim: Dimension, nullability: Nullabilit
9596
DType::List(Arc::new(polygon), nullability)
9697
}
9798

98-
/// Validate `dtype` is `List<List<List<coordinate-struct>>>` and return its [`Dimension`].
99+
/// Validate that `dtype` equals [`multipolygon_storage_dtype`] for its [`Dimension`] and top-level
100+
/// nullability, and return that dimension. As in GeoArrow, only the outer `List` can be nullable.
99101
pub(crate) fn multipolygon_dimension(dtype: &DType) -> VortexResult<Dimension> {
100102
let DType::List(polygon, _) = dtype else {
101103
vortex_bail!("multipolygon storage must be a List of polygons, was {dtype}");
@@ -106,7 +108,15 @@ pub(crate) fn multipolygon_dimension(dtype: &DType) -> VortexResult<Dimension> {
106108
let DType::List(coords, _) = ring.as_ref() else {
107109
vortex_bail!("multipolygon ring storage must be a List of coordinates, was {ring}");
108110
};
109-
coordinate_dimension(coords)
111+
let dimension = coordinate_dimension(coords)?;
112+
113+
vortex_ensure_eq!(
114+
*dtype,
115+
multipolygon_storage_dtype(dimension, dtype.nullability()),
116+
"multipolygon storage can be nullable only at its outer List"
117+
);
118+
119+
Ok(dimension)
110120
}
111121

112122
static ARROW_MULTIPOLYGON: CachedId = CachedId::new(MultiPolygonType::NAME);

‎vortex-spatial/src/extension/polygon.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ use vortex_error::VortexError;
4747
use vortex_error::VortexResult;
4848
use vortex_error::vortex_bail;
4949
use vortex_error::vortex_ensure;
50+
use vortex_error::vortex_ensure_eq;
5051
use vortex_error::vortex_err;
5152
use vortex_session::registry::CachedId;
5253
use vortex_session::registry::Id;
@@ -100,15 +101,24 @@ pub(crate) fn polygon_storage_dtype(dim: Dimension, nullability: Nullability) ->
100101
DType::List(Arc::new(ring), nullability)
101102
}
102103

103-
/// Validate `dtype` is `List<List<coordinate-struct>>` and return its [`Dimension`].
104+
/// Validate that `dtype` equals [`polygon_storage_dtype`] for its [`Dimension`] and top-level
105+
/// nullability, and return that dimension. As in GeoArrow, only the outer `List` can be nullable.
104106
pub(crate) fn polygon_dimension(dtype: &DType) -> VortexResult<Dimension> {
105107
let DType::List(ring, _) = dtype else {
106108
vortex_bail!("polygon storage must be a List of rings, was {dtype}");
107109
};
108110
let DType::List(coords, _) = ring.as_ref() else {
109111
vortex_bail!("polygon ring storage must be a List of coordinates, was {ring}");
110112
};
111-
coordinate_dimension(coords)
113+
let dimension = coordinate_dimension(coords)?;
114+
115+
vortex_ensure_eq!(
116+
*dtype,
117+
polygon_storage_dtype(dimension, dtype.nullability()),
118+
"polygon storage can be nullable only at its outer List"
119+
);
120+
121+
Ok(dimension)
112122
}
113123

114124
static ARROW_POLYGON: CachedId = CachedId::new(PolygonType::NAME);

0 commit comments

Comments
 (0)