Skip to content

Commit 488298d

Browse files
committed
feat: Variant Support
1 parent 01a6a43 commit 488298d

19 files changed

Lines changed: 720 additions & 27 deletions

File tree

‎crates/catalog/glue/src/schema.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,13 @@ impl SchemaVisitor for GlueSchemaBuilder {
182182

183183
Ok(glue_type)
184184
}
185+
186+
fn variant(&mut self, _v: &iceberg::spec::VariantType) -> Result<Self::T> {
187+
Err(Error::new(
188+
ErrorKind::FeatureUnsupported,
189+
"Conversion from VariantType is not supported for Glue",
190+
))
191+
}
185192
}
186193

187194
#[cfg(test)]

‎crates/catalog/hms/src/schema.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,13 @@ impl SchemaVisitor for HiveSchemaBuilder {
139139

140140
Ok(hive_type)
141141
}
142+
143+
fn variant(&mut self, _v: &iceberg::spec::VariantType) -> Result<Self::T> {
144+
Err(Error::new(
145+
ErrorKind::FeatureUnsupported,
146+
"Conversion from VariantType is not supported for HMS",
147+
))
148+
}
142149
}
143150

144151
#[cfg(test)]

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use crate::io::FileIO;
3333
use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile};
3434
use crate::spec::{
3535
DataContentType, Datum, ListType, MapType, NestedField, NestedFieldRef, PartnerAccessor,
36-
PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type,
36+
PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type, VariantType,
3737
visit_schema_with_partner,
3838
};
3939
use crate::{Error, ErrorKind, Result};
@@ -527,6 +527,10 @@ impl SchemaWithPartnerVisitor<ArrayRef> for EqDelColumnProcessor<'_> {
527527
fn primitive(&mut self, _primitive: &PrimitiveType, _partner: &ArrayRef) -> Result<()> {
528528
Ok(())
529529
}
530+
531+
fn variant(&mut self, _v: &VariantType, _partner: &ArrayRef) -> Result<()> {
532+
Ok(())
533+
}
530534
}
531535

532536
struct EqDelRecordBatchPartnerAccessor;

‎crates/iceberg/src/arrow/nan_val_cnt_visitor.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use crate::Result;
2828
use crate::arrow::{ArrowArrayAccessor, FieldMatchMode};
2929
use crate::spec::{
3030
ListType, MapType, NestedFieldRef, PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor,
31-
StructType, visit_struct_with_partner,
31+
StructType, VariantType, visit_struct_with_partner,
3232
};
3333

3434
macro_rules! cast_and_update_cnt_map {
@@ -122,6 +122,10 @@ impl SchemaWithPartnerVisitor<ArrayRef> for NanValueCountVisitor {
122122
Ok(())
123123
}
124124

125+
fn variant(&mut self, _v: &VariantType, _col: &ArrayRef) -> Result<Self::T> {
126+
Ok(())
127+
}
128+
125129
fn after_struct_field(&mut self, field: &NestedFieldRef, partner: &ArrayRef) -> Result<()> {
126130
let field_id = field.id;
127131
count_float_nans!(partner, self, field_id);

‎crates/iceberg/src/arrow/reader.rs‎

Lines changed: 151 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,10 @@ impl ArrowReader {
682682
Self::include_leaf_field_id(&map_type.key_field, field_ids);
683683
Self::include_leaf_field_id(&map_type.value_field, field_ids);
684684
}
685+
// Variant is a leaf type for Parquet projection purposes (like a primitive).
686+
Type::Variant(_) => {
687+
field_ids.push(field.id);
688+
}
685689
}
686690
}
687691

@@ -754,8 +758,46 @@ impl ArrowReader {
754758
arrow_schema: &ArrowSchemaRef,
755759
type_promotion_is_valid: fn(Option<&PrimitiveType>, Option<&PrimitiveType>) -> bool,
756760
) -> Result<ProjectionMask> {
757-
let mut column_map = HashMap::new();
761+
// Maps field_id → leaf column indices. Vec because variant contributes two
762+
// leaves (metadata + value) under a single field ID.
763+
let mut column_map: HashMap<i32, Vec<usize>> = HashMap::new();
758764
let fields = arrow_schema.fields();
765+
// HashSet for O(1) membership checks instead of O(n) slice scans.
766+
let leaf_field_id_set: HashSet<i32> = leaf_field_ids.iter().copied().collect();
767+
768+
// Variant fields are an Iceberg leaf type but a Parquet GROUP. Their
769+
// sub-fields (metadata, value) carry no embedded field IDs — only the
770+
// parent group has the field ID. filter_leaves therefore never finds
771+
// them via the standard field-ID scan below.
772+
//
773+
// Java's PruneColumns.variant() simply returns the group as-is with no
774+
// type-compatibility check (isStruct() also short-circuits on isVariantType()).
775+
// We replicate that here: pre-scan top-level Arrow struct fields whose
776+
// field ID resolves to Type::Variant and record all their sub-fields so
777+
// the second filter_leaves can include them directly.
778+
let mut variant_sub_fields: HashMap<FieldRef, i32> = HashMap::new();
779+
for top_field in fields.iter() {
780+
let Some(field_id) = top_field
781+
.metadata()
782+
.get(PARQUET_FIELD_ID_META_KEY)
783+
.and_then(|s| i32::from_str(s).ok())
784+
else {
785+
continue;
786+
};
787+
if !leaf_field_id_set.contains(&field_id) {
788+
continue;
789+
}
790+
let Some(iceberg_field) = iceberg_schema_of_task.field_by_id(field_id) else {
791+
continue;
792+
};
793+
if let Type::Variant(_) = iceberg_field.field_type.as_ref()
794+
&& let DataType::Struct(sub_fields) = top_field.data_type()
795+
{
796+
for sub_field in sub_fields {
797+
variant_sub_fields.insert(sub_field.clone(), field_id);
798+
}
799+
}
800+
}
759801

760802
// Pre-project only the fields that have been selected, possibly avoiding converting
761803
// some Arrow types that are not yet supported.
@@ -767,14 +809,22 @@ impl ArrowReader {
767809
.and_then(|field_id| i32::from_str(field_id).ok())
768810
.is_some_and(|field_id| {
769811
projected_fields.insert((*f).clone(), field_id);
770-
leaf_field_ids.contains(&field_id)
812+
leaf_field_id_set.contains(&field_id)
771813
})
772814
}),
773815
arrow_schema.metadata().clone(),
774816
);
775817
let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?;
776818

777819
fields.filter_leaves(|idx, field| {
820+
// Variant sub-fields: parent group carries the field ID, not the leaf.
821+
// Skip type-promotion check — Type::Variant is not a primitive type
822+
// (matches Java's PruneColumns.variant() which returns the group unchanged).
823+
if let Some(&variant_field_id) = variant_sub_fields.get(field) {
824+
column_map.entry(variant_field_id).or_default().push(idx);
825+
return true;
826+
}
827+
778828
let Some(field_id) = projected_fields.get(field).cloned() else {
779829
return false;
780830
};
@@ -796,16 +846,16 @@ impl ArrowReader {
796846
return false;
797847
}
798848

799-
column_map.insert(field_id, idx);
849+
column_map.entry(field_id).or_default().push(idx);
800850
true
801851
});
802852

803853
// Schema evolution: New columns may not exist in old Parquet files.
804854
// We only project existing columns; RecordBatchTransformer adds default/NULL values.
805855
let mut indices = vec![];
806856
for field_id in leaf_field_ids {
807-
if let Some(col_idx) = column_map.get(field_id) {
808-
indices.push(*col_idx);
857+
if let Some(col_indices) = column_map.get(field_id) {
858+
indices.extend_from_slice(col_indices);
809859
}
810860
}
811861

@@ -1835,7 +1885,8 @@ mod tests {
18351885
use crate::io::FileIO;
18361886
use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream};
18371887
use crate::spec::{
1838-
DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type,
1888+
DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef,
1889+
Type, VariantType,
18391890
};
18401891

18411892
fn table_schema_simple() -> SchemaRef {
@@ -2003,6 +2054,100 @@ message schema {
20032054
assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0]));
20042055
}
20052056

2057+
/// Variant fields are an Iceberg leaf type but a Parquet GROUP whose sub-fields carry
2058+
/// no embedded field IDs. The projection mask must include both metadata and value
2059+
/// leaves when the variant field ID is requested, and must not drop the variant when
2060+
/// it is projected alongside ordinary primitive columns.
2061+
#[test]
2062+
fn test_arrow_projection_mask_variant() {
2063+
// Iceberg schema: c1 (String, id=1) + v (Variant, id=2)
2064+
let schema = Arc::new(
2065+
Schema::builder()
2066+
.with_schema_id(1)
2067+
.with_fields(vec![
2068+
NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(),
2069+
NestedField::required(2, "v", Type::Variant(VariantType)).into(),
2070+
])
2071+
.build()
2072+
.unwrap(),
2073+
);
2074+
2075+
// Arrow schema: c1 with field ID 1; v as Struct(metadata: Binary, value: Binary)
2076+
// with field ID 2 on the struct but NO field IDs on the sub-fields — that is the
2077+
// Parquet variant wire format.
2078+
let arrow_schema = Arc::new(ArrowSchema::new(vec![
2079+
Field::new("c1", DataType::Utf8, false).with_metadata(HashMap::from([(
2080+
PARQUET_FIELD_ID_META_KEY.to_string(),
2081+
"1".to_string(),
2082+
)])),
2083+
Field::new(
2084+
"v",
2085+
DataType::Struct(arrow_schema::Fields::from(vec![
2086+
Arc::new(Field::new("metadata", DataType::Binary, false)),
2087+
Arc::new(Field::new("value", DataType::Binary, false)),
2088+
])),
2089+
false,
2090+
)
2091+
.with_metadata(HashMap::from([(
2092+
PARQUET_FIELD_ID_META_KEY.to_string(),
2093+
"2".to_string(),
2094+
)])),
2095+
]));
2096+
2097+
// Parquet message: c1 is leaf 0; variant sub-fields metadata=leaf 1, value=leaf 2.
2098+
// No field IDs on sub-leaves — matching the real Iceberg/Spark-written variant format.
2099+
let message_type = "
2100+
message schema {
2101+
required binary c1 (STRING) = 1;
2102+
required group v = 2 {
2103+
required binary metadata;
2104+
required binary value;
2105+
}
2106+
}
2107+
";
2108+
let parquet_type = parse_message_type(message_type).expect("should parse schema");
2109+
let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type));
2110+
2111+
// Both fields: all three leaves must be included.
2112+
let mask = ArrowReader::get_arrow_projection_mask(
2113+
&[1, 2],
2114+
&schema,
2115+
&parquet_schema,
2116+
&arrow_schema,
2117+
false,
2118+
)
2119+
.expect("projection mask for c1 + v");
2120+
assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0, 1, 2]));
2121+
2122+
// Variant only: leaves 1 (metadata) and 2 (value) must be included.
2123+
let mask_variant_only = ArrowReader::get_arrow_projection_mask(
2124+
&[2],
2125+
&schema,
2126+
&parquet_schema,
2127+
&arrow_schema,
2128+
false,
2129+
)
2130+
.expect("projection mask for v only");
2131+
assert_eq!(
2132+
mask_variant_only,
2133+
ProjectionMask::leaves(&parquet_schema, vec![1, 2]),
2134+
);
2135+
2136+
// Primitive only: leaf 0 (c1) must be included; variant NOT included.
2137+
let mask_primitive_only = ArrowReader::get_arrow_projection_mask(
2138+
&[1],
2139+
&schema,
2140+
&parquet_schema,
2141+
&arrow_schema,
2142+
false,
2143+
)
2144+
.expect("projection mask for c1 only");
2145+
assert_eq!(
2146+
mask_primitive_only,
2147+
ProjectionMask::leaves(&parquet_schema, vec![0]),
2148+
);
2149+
}
2150+
20062151
#[tokio::test]
20072152
async fn test_kleene_logic_or_behaviour() {
20082153
// a IS NULL OR a = 'foo'

‎crates/iceberg/src/arrow/schema.rs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use crate::error::Result;
3535
use crate::spec::decimal_utils::i128_from_be_bytes;
3636
use crate::spec::{
3737
Datum, FIRST_FIELD_ID, ListType, MapType, NestedField, NestedFieldRef, PrimitiveLiteral,
38-
PrimitiveType, Schema, SchemaVisitor, StructType, Type,
38+
PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType,
3939
};
4040
use crate::{Error, ErrorKind};
4141

@@ -689,6 +689,16 @@ impl SchemaVisitor for ToArrowSchemaConverter {
689689
}
690690
}
691691
}
692+
693+
fn variant(&mut self, _v: &VariantType) -> crate::Result<ArrowSchemaOrFieldOrType> {
694+
// Variant is stored as a struct with two required binary fields (no field IDs on sub-fields).
695+
// Uses Binary (not LargeBinary) matching the Parquet BINARY primitive directly.
696+
let metadata_field = Field::new("metadata", DataType::Binary, false);
697+
let value_field = Field::new("value", DataType::Binary, false);
698+
Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(
699+
vec![metadata_field, value_field].into(),
700+
)))
701+
}
692702
}
693703

694704
/// Convert iceberg schema to an arrow schema.
@@ -1697,6 +1707,15 @@ mod tests {
16971707
simple_field("map", map, false, "16"),
16981708
simple_field("struct", r#struct, false, "17"),
16991709
simple_field("uuid", DataType::FixedSizeBinary(16), false, "30"),
1710+
simple_field(
1711+
"v",
1712+
DataType::Struct(Fields::from(vec![
1713+
Field::new("metadata", DataType::Binary, false),
1714+
Field::new("value", DataType::Binary, false),
1715+
])),
1716+
true,
1717+
"31",
1718+
),
17001719
])
17011720
}
17021721

@@ -1880,6 +1899,12 @@ mod tests {
18801899
"name":"uuid",
18811900
"required":true,
18821901
"type":"uuid"
1902+
},
1903+
{
1904+
"id":31,
1905+
"name":"v",
1906+
"required":false,
1907+
"type":"variant"
18831908
}
18841909
],
18851910
"identifier-field-ids":[]

‎crates/iceberg/src/arrow/value.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use uuid::Uuid;
3030
use super::get_field_id_from_metadata;
3131
use crate::spec::{
3232
ListType, Literal, Map, MapType, NestedField, PartnerAccessor, PrimitiveLiteral, PrimitiveType,
33-
SchemaWithPartnerVisitor, Struct, StructType, Type, visit_struct_with_partner,
33+
SchemaWithPartnerVisitor, Struct, StructType, Type, VariantType, visit_struct_with_partner,
3434
visit_type_with_partner,
3535
};
3636
use crate::{Error, ErrorKind, Result};
@@ -426,6 +426,13 @@ impl SchemaWithPartnerVisitor<ArrayRef> for ArrowArrayToIcebergStructConverter {
426426
}
427427
}
428428
}
429+
430+
fn variant(&mut self, _v: &VariantType, _partner: &ArrayRef) -> Result<Vec<Option<Literal>>> {
431+
Err(Error::new(
432+
ErrorKind::FeatureUnsupported,
433+
"Converting variant Arrow array to Iceberg literal is not supported yet",
434+
))
435+
}
429436
}
430437

431438
/// Defines how Arrow fields are matched with Iceberg fields when converting data.

0 commit comments

Comments
 (0)