Skip to content

Commit f136c36

Browse files
committed
fix(arrow): address final review on nested field-id promotion
- Add a reader-level test that writes an old nested schema to Parquet and reads it back through an evolved one, pinning that parquet-rs attaches field ids to struct children, list elements, and map keys and values - Reject duplicate source field ids and nested sources read as a non-nested type when the plan is built - Report non-struct map entries as DataInvalid - Pin the id-less same-type reorder fallback with a test until #1845 - Assert map data types, since map array equality skips field names - Point at #3378 where the projection mask skips all-missing structs
1 parent ee881aa commit f136c36

2 files changed

Lines changed: 195 additions & 12 deletions

File tree

‎crates/iceberg/src/arrow/reader/projection.rs‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@ impl ArrowReader {
221221

222222
// Schema evolution: New columns may not exist in old Parquet files.
223223
// We only project existing columns; RecordBatchTransformer adds default/NULL values.
224+
// If other leaves are selected, a struct whose requested leaves are all missing gets no
225+
// column and reads as NULL (#3378).
224226
let mut indices = vec![];
225227
for field_id in leaf_field_ids {
226228
if let Some(col_idx) = column_map.get(field_id) {
@@ -2046,4 +2048,126 @@ message schema {
20462048
.collect();
20472049
assert_eq!(ids, vec![2, 3]);
20482050
}
2051+
2052+
/// Reads a file written with an older nested schema through an evolved one. This relies
2053+
/// on parquet-rs attaching `PARQUET:field_id` to struct children, list elements, and map
2054+
/// keys and values, which the hand-built fixtures in `record_batch_transformer` assume.
2055+
#[tokio::test]
2056+
async fn test_read_evolved_nested_fields_by_field_id() {
2057+
use serde_json::{Value, json};
2058+
2059+
use crate::arrow::schema_to_arrow_schema;
2060+
use crate::spec::PrimitiveType::{Int, Long};
2061+
use crate::spec::{ListType, MapType};
2062+
2063+
fn struct_of(fields: &[(i32, &str, PrimitiveType)]) -> Type {
2064+
Type::Struct(StructType::new(
2065+
fields
2066+
.iter()
2067+
.map(|(id, name, ty)| {
2068+
NestedField::optional(*id, *name, Type::Primitive(ty.clone())).into()
2069+
})
2070+
.collect(),
2071+
))
2072+
}
2073+
fn schema(s: Type, element: Type, value: Type) -> Schema {
2074+
let string = Type::Primitive(PrimitiveType::String);
2075+
Schema::builder()
2076+
.with_fields(vec![
2077+
NestedField::required(1, "id", Type::Primitive(Int)).into(),
2078+
NestedField::optional(2, "s", s).into(),
2079+
NestedField::optional(
2080+
3,
2081+
"l",
2082+
Type::List(ListType::new(
2083+
NestedField::list_element(7, element, true).into(),
2084+
)),
2085+
)
2086+
.into(),
2087+
NestedField::optional(
2088+
4,
2089+
"m",
2090+
Type::Map(MapType::required(9, string, 10, value)),
2091+
)
2092+
.into(),
2093+
])
2094+
.build()
2095+
.unwrap()
2096+
}
2097+
fn to_batch(schema: &Schema, rows: Value) -> RecordBatch {
2098+
let fields = schema_to_arrow_schema(schema).unwrap().fields().to_vec();
2099+
serde_arrow::to_record_batch(&fields, &rows).unwrap()
2100+
}
2101+
2102+
let old_schema = schema(
2103+
struct_of(&[(5, "a", Int), (6, "b", PrimitiveType::String)]),
2104+
struct_of(&[(8, "x", Int)]),
2105+
struct_of(&[(11, "x", Int)]),
2106+
);
2107+
// Reorders `s`, adds children before, between, and after existing ones, and
2108+
// promotes int to long.
2109+
let new_schema = Arc::new(schema(
2110+
struct_of(&[
2111+
(6, "b", PrimitiveType::String),
2112+
(12, "c", Int),
2113+
(5, "a", Long),
2114+
]),
2115+
struct_of(&[(13, "y", Int), (8, "x", Int)]),
2116+
struct_of(&[(11, "x", Long), (14, "y", Int)]),
2117+
));
2118+
let old_batch = to_batch(
2119+
&old_schema,
2120+
json!([
2121+
{"id": 1, "s": {"a": 1, "b": "p"}, "l": [{"x": 10}, {"x": 11}], "m": {"k1": {"x": 100}}},
2122+
{"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200}}},
2123+
{"id": 3, "s": {"a": 3, "b": null}, "l": [{"x": 30}], "m": null},
2124+
]),
2125+
);
2126+
let expected = to_batch(
2127+
&new_schema,
2128+
json!([
2129+
{
2130+
"id": 1,
2131+
"s": {"b": "p", "c": null, "a": 1},
2132+
"l": [{"y": null, "x": 10}, {"y": null, "x": 11}],
2133+
"m": {"k1": {"x": 100, "y": null}},
2134+
},
2135+
{"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200, "y": null}}},
2136+
{"id": 3, "s": {"b": null, "c": null, "a": 3}, "l": [{"y": null, "x": 30}], "m": null},
2137+
]),
2138+
);
2139+
2140+
let tmp_dir = TempDir::new().unwrap();
2141+
let file_path = format!("{}/old.parquet", tmp_dir.path().to_str().unwrap());
2142+
let mut writer =
2143+
ArrowWriter::try_new(File::create(&file_path).unwrap(), old_batch.schema(), None)
2144+
.unwrap();
2145+
writer.write(&old_batch).unwrap();
2146+
writer.close().unwrap();
2147+
2148+
let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2149+
let tasks = Box::pin(futures::stream::iter(
2150+
vec![Ok(FileScanTask::builder()
2151+
.with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2152+
.with_start(0)
2153+
.with_length(0)
2154+
.with_data_file_path(file_path)
2155+
.with_data_file_format(DataFileFormat::Parquet)
2156+
.with_schema(new_schema)
2157+
.with_project_field_ids(vec![1, 2, 3, 4])
2158+
.with_case_sensitive(false)
2159+
.build()
2160+
.unwrap())]
2161+
.into_iter(),
2162+
)) as FileScanTaskStream;
2163+
2164+
let result = reader
2165+
.read(tasks)
2166+
.unwrap()
2167+
.stream()
2168+
.try_collect::<Vec<RecordBatch>>()
2169+
.await
2170+
.unwrap();
2171+
assert_eq!(result, vec![expected]);
2172+
}
20492173
}

‎crates/iceberg/src/arrow/record_batch_transformer.rs‎

Lines changed: 71 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -283,16 +283,15 @@ impl PromotePlan {
283283
sorted: *sorted,
284284
})
285285
}
286-
_ => Err(Error::new(
287-
ErrorKind::Unexpected,
288-
format!(
289-
"expected struct-typed map entries, got {source_entries:?} and {target_entries:?}"
290-
),
286+
_ => Err(invalid_data!(
287+
"expected struct-typed map entries, got {source_entries:?} and {target_entries:?}"
291288
)),
292289
}
293290
}
294291
(_, DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_))
295-
| (_, DataType::Map(_, _)) => {
292+
| (_, DataType::Map(_, _))
293+
| (DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_), _)
294+
| (DataType::Map(_, _), _) => {
296295
Err(invalid_data!("cannot promote {source:?} to {target:?}"))
297296
}
298297
_ => Ok(PromotePlan::Cast(target.clone())),
@@ -307,14 +306,16 @@ impl PromotePlan {
307306
) -> Result<Vec<ChildPlan>> {
308307
let mut source_by_id = HashMap::with_capacity(source_fields.len());
309308
for (idx, field) in source_fields.iter().enumerate() {
310-
if let Some(id) = try_get_field_id_from_metadata(field)? {
311-
source_by_id.insert(id, idx);
309+
if let Some(id) = try_get_field_id_from_metadata(field)?
310+
&& source_by_id.insert(id, idx).is_some()
311+
{
312+
return Err(invalid_data!("duplicate field id {id} in struct"));
312313
}
313314
}
314-
// Name mapping only assigns top-level ids (#1845). Fully id-less children
315-
// match by position when the counts line up, and `build` checks each pair.
316-
// A same-type reorder cannot be detected without ids. Any missing id
317-
// errors instead of nulling that child.
315+
// Name mapping only assigns top-level ids, so fully id-less children match
316+
// by position when the counts line up, and `build` checks each pair. This is
317+
// a compatibility fallback until recursive name mapping (#1845): a same-type
318+
// reorder reads positionally. Any missing id errors instead of nulling that child.
318319
if source_by_id.len() != source_fields.len() {
319320
if source_by_id.is_empty() && source_fields.len() == target_fields.len() {
320321
return source_fields
@@ -1633,6 +1634,7 @@ mod test {
16331634
None,
16341635
false,
16351636
);
1637+
assert_eq!(out.data_type(), expected.data_type());
16361638
assert_eq!(out.as_map(), &expected);
16371639
}
16381640

@@ -1694,6 +1696,30 @@ mod test {
16941696
assert!(err.to_string().contains("do not all have field ids"));
16951697
}
16961698

1699+
#[test]
1700+
fn promote_rejects_invalid_source_at_build() {
1701+
let duplicate_ids = DataType::Struct(Fields::from(vec![
1702+
field_with_id("x", DataType::Int32, true, 5),
1703+
field_with_id("y", DataType::Int32, true, 5),
1704+
]));
1705+
let map_of =
1706+
|entries| DataType::Map(Arc::new(Field::new("entries", entries, false)), false);
1707+
for (source, target, message) in [
1708+
(duplicate_ids, evolved_struct_type(), "duplicate field id 5"),
1709+
(unevolved_struct_type(), DataType::Int32, "cannot promote"),
1710+
(
1711+
map_of(DataType::Int32),
1712+
map_of(DataType::Int64),
1713+
"expected struct-typed map entries",
1714+
),
1715+
] {
1716+
let err =
1717+
PromotePlan::build(&source, &target, &empty_schema(), &HashMap::new()).unwrap_err();
1718+
assert_eq!(err.kind(), ErrorKind::DataInvalid);
1719+
assert!(err.to_string().contains(message), "{err}");
1720+
}
1721+
}
1722+
16971723
#[test]
16981724
fn promote_required_nested_field_absent_without_default_errors() {
16991725
let schema = Schema::builder()
@@ -1896,9 +1922,42 @@ mod test {
18961922
None,
18971923
false,
18981924
);
1925+
// Map array equality skips the entries field name.
1926+
assert_eq!(out.data_type(), expected.data_type());
18991927
assert_eq!(out.as_map(), &expected);
19001928
}
19011929

1930+
#[test]
1931+
fn promote_idless_same_type_reorder_stays_positional() {
1932+
// Without ids a same-type reorder is undetectable; #1845 replaces this fallback.
1933+
let source = Arc::new(StructArray::new(
1934+
Fields::from(vec![
1935+
Field::new("a", DataType::Int32, true),
1936+
Field::new("b", DataType::Int32, true),
1937+
]),
1938+
vec![
1939+
Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1940+
Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef,
1941+
],
1942+
None,
1943+
)) as ArrayRef;
1944+
let fields = Fields::from(vec![
1945+
field_with_id("b", DataType::Int32, true, 6),
1946+
field_with_id("a", DataType::Int32, true, 5),
1947+
]);
1948+
1949+
let out = promote(&source, &DataType::Struct(fields.clone()), &empty_schema()).unwrap();
1950+
let expected = StructArray::new(
1951+
fields,
1952+
vec![
1953+
Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1954+
Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef,
1955+
],
1956+
None,
1957+
);
1958+
assert_eq!(out.as_struct(), &expected);
1959+
}
1960+
19021961
#[test]
19031962
fn promote_large_list_file_column_to_list() {
19041963
let element = Arc::new(field_with_id("element", DataType::Int32, true, 3));

0 commit comments

Comments
 (0)