Skip to content

Commit 993334b

Browse files
committed
Keep binary map children on the validating cast path
Matching map keys and values by position newly reaches children whose names differ between the file and the table schema. Coercing a binary child to a string type there would be a regression: the Parquet reader only validates UTF-8 for columns carrying the `UTF8` logical annotation, which a binary column by definition does not, so the read returns an invalid string array where before the (validating) cast raised `Encountered non UTF-8 data`. Thread a `binary_to_string` flag through the coercion helpers and pass `false` below a map, so map children only get conversions that cannot fabricate UTF-8 validity. Top-level fields, struct children and list children keep their existing behaviour. The same unvalidated conversion is reachable today for top-level and struct fields, which this does not change; that is tracked separately. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01PL624RiDwff7RVJ2BKSvS5
1 parent adad560 commit 993334b

1 file changed

Lines changed: 166 additions & 25 deletions

File tree

‎datafusion/datasource-parquet/src/schema_coercion.rs‎

Lines changed: 166 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,16 @@
2222
//! These helpers are independent of the [`ParquetFormat`](crate::file_format::ParquetFormat)
2323
//! type and several have been re-exported at the crate root for use by
2424
//! callers outside the format implementation.
25+
//!
26+
//! # Binary to string coercion
27+
//!
28+
//! Coercing a binary file column to a string type makes the Parquet reader
29+
//! build the string array itself. It only validates UTF-8 for columns carrying
30+
//! the `UTF8` logical annotation, which such a column by definition does not,
31+
//! so invalid bytes surface as an invalid string array rather than an error.
32+
//! That is long-standing behaviour for top-level and struct fields; it is not
33+
//! extended to map children, whose positional matching is newer, so binary map
34+
//! values keep going through the validating cast instead.
2535
2636
use std::cell::RefCell;
2737
use std::collections::{HashMap, HashSet};
@@ -66,7 +76,8 @@ pub fn apply_file_schema_type_coercions(
6676
table_schema: &Schema,
6777
file_schema: &Schema,
6878
) -> Option<Schema> {
69-
let fields = coerce_fields_by_name(table_schema.fields(), file_schema.fields())?;
79+
let fields =
80+
coerce_fields_by_name(table_schema.fields(), file_schema.fields(), true)?;
7081
Some(Schema::new_with_metadata(
7182
fields,
7283
file_schema.metadata.clone(),
@@ -78,7 +89,11 @@ pub fn apply_file_schema_type_coercions(
7889
/// File fields with no counterpart in `table_fields` are kept unchanged and
7990
/// table fields missing from the file are ignored. Returns `None` if no field
8091
/// changed.
81-
fn coerce_fields_by_name(table_fields: &Fields, file_fields: &Fields) -> Option<Fields> {
92+
fn coerce_fields_by_name(
93+
table_fields: &Fields,
94+
file_fields: &Fields,
95+
binary_to_string: bool,
96+
) -> Option<Fields> {
8297
// Create a mapping of table field names to their data types for fast lookup
8398
let table_types: HashMap<_, _> = table_fields
8499
.iter()
@@ -87,7 +102,7 @@ fn coerce_fields_by_name(table_fields: &Fields, file_fields: &Fields) -> Option<
87102

88103
coerce_fields(file_fields, |_, field| {
89104
let table_type = table_types.get(field.name())?;
90-
coerce_data_type(table_type, field.data_type())
105+
coerce_data_type(table_type, field.data_type(), binary_to_string)
91106
.map(|new_type| field_with_new_type(field, new_type))
92107
})
93108
}
@@ -127,40 +142,53 @@ fn coerce_fields(
127142

128143
/// Coerce `file_type` towards `table_type`, recursing into nested types.
129144
///
145+
/// `binary_to_string` allows binary file types to be read as string types.
146+
/// Parquet only validates UTF-8 when the column carries the `UTF8` logical
147+
/// annotation, which a binary column by definition does not, so such a
148+
/// conversion hands back an unvalidated string array; see the module docs.
149+
///
130150
/// Returns the new type for the file field, or `None` if no transformation
131151
/// is needed (including when the two types are unrelated).
132-
fn coerce_data_type(table_type: &DataType, file_type: &DataType) -> Option<DataType> {
152+
fn coerce_data_type(
153+
table_type: &DataType,
154+
file_type: &DataType,
155+
binary_to_string: bool,
156+
) -> Option<DataType> {
133157
use DataType::*;
134158
match (table_type, file_type) {
135159
// table schema uses string type, coerce the file schema to use string type
136-
(Utf8, Binary | LargeBinary | BinaryView) => Some(Utf8),
160+
(Utf8, Binary | LargeBinary | BinaryView) if binary_to_string => Some(Utf8),
137161
// table schema uses large string type, coerce the file schema to use large string type
138-
(LargeUtf8, Binary | LargeBinary | BinaryView) => Some(LargeUtf8),
162+
(LargeUtf8, Binary | LargeBinary | BinaryView) if binary_to_string => {
163+
Some(LargeUtf8)
164+
}
139165
// table schema uses string view type, coerce the file schema to use view type
140-
(Utf8View, Binary | LargeBinary | BinaryView | Utf8 | LargeUtf8) => {
166+
(Utf8View, Binary | LargeBinary | BinaryView) if binary_to_string => {
141167
Some(Utf8View)
142168
}
169+
(Utf8View, Utf8 | LargeUtf8) => Some(Utf8View),
143170
(BinaryView, Binary | LargeBinary) => Some(BinaryView),
144171
// Struct children match by name
145172
(Struct(table_fields), Struct(file_fields)) => {
146-
coerce_fields_by_name(table_fields, file_fields).map(Struct)
173+
coerce_fields_by_name(table_fields, file_fields, binary_to_string).map(Struct)
147174
}
148175
// List-like children match by position, regardless of their names.
149176
// The container kind and FixedSizeList width always come from the file.
150177
(List(table_child), List(file_child)) => {
151-
coerce_child(table_child, file_child).map(List)
178+
coerce_child(table_child, file_child, binary_to_string).map(List)
152179
}
153180
(LargeList(table_child), LargeList(file_child)) => {
154-
coerce_child(table_child, file_child).map(LargeList)
181+
coerce_child(table_child, file_child, binary_to_string).map(LargeList)
155182
}
156183
(ListView(table_child), ListView(file_child)) => {
157-
coerce_child(table_child, file_child).map(ListView)
184+
coerce_child(table_child, file_child, binary_to_string).map(ListView)
158185
}
159186
(LargeListView(table_child), LargeListView(file_child)) => {
160-
coerce_child(table_child, file_child).map(LargeListView)
187+
coerce_child(table_child, file_child, binary_to_string).map(LargeListView)
161188
}
162189
(FixedSizeList(table_child, _), FixedSizeList(file_child, size)) => {
163-
coerce_child(table_child, file_child).map(|child| FixedSizeList(child, *size))
190+
coerce_child(table_child, file_child, binary_to_string)
191+
.map(|child| FixedSizeList(child, *size))
164192
}
165193
// Map keys and values match by position: Parquet always names them
166194
// `key`/`value` while Arrow producers commonly use `keys`/`values`.
@@ -174,13 +202,26 @@ fn coerce_data_type(table_type: &DataType, file_type: &DataType) -> Option<DataT
174202

175203
/// Coerce a single nested child field, keeping everything but its data type
176204
/// from `file_child`.
177-
fn coerce_child(table_child: &FieldRef, file_child: &FieldRef) -> Option<FieldRef> {
178-
coerce_data_type(table_child.data_type(), file_child.data_type())
179-
.map(|new_type| field_with_new_type(file_child, new_type))
205+
fn coerce_child(
206+
table_child: &FieldRef,
207+
file_child: &FieldRef,
208+
binary_to_string: bool,
209+
) -> Option<FieldRef> {
210+
coerce_data_type(
211+
table_child.data_type(),
212+
file_child.data_type(),
213+
binary_to_string,
214+
)
215+
.map(|new_type| field_with_new_type(file_child, new_type))
180216
}
181217

182218
/// Coerce the `entries` struct of a [`DataType::Map`], matching the key and
183219
/// value children by position.
220+
///
221+
/// Binary children are never read as strings here: matching by position newly
222+
/// reaches map children whose names differ, and turning those into unvalidated
223+
/// string arrays would be a regression over leaving them to the (validating)
224+
/// cast. See the module docs.
184225
fn coerce_map_entries(
185226
table_entries: &FieldRef,
186227
file_entries: &FieldRef,
@@ -195,7 +236,7 @@ fn coerce_map_entries(
195236
}
196237

197238
let fields = coerce_fields(file_fields, |idx, file_child| {
198-
coerce_child(&table_fields[idx], file_child)
239+
coerce_child(&table_fields[idx], file_child, false)
199240
})?;
200241

201242
Some(field_with_new_type(file_entries, DataType::Struct(fields)))
@@ -629,7 +670,7 @@ mod tests {
629670
"key_value",
630671
vec![
631672
Field::new("key", DataType::Utf8, false),
632-
Field::new("value", DataType::Binary, true),
673+
Field::new("value", DataType::Utf8, true),
633674
],
634675
false,
635676
)
@@ -684,15 +725,18 @@ mod tests {
684725
"m",
685726
"entries",
686727
Field::new("keys", DataType::Utf8View, false),
687-
Field::new("values", DataType::Utf8, true),
728+
Field::new("values", DataType::Utf8View, true),
688729
false,
689730
true,
690731
)]);
732+
// The key is matched by position and coerced to a view, while the
733+
// binary value is left to the validating cast (see
734+
// `coerce_map_entries`)
691735
let expected = Schema::new(vec![Field::new_map(
692736
"m",
693737
"key_value",
694738
Field::new("key", DataType::Utf8View, false),
695-
Field::new("value", DataType::Utf8, true),
739+
Field::new("value", DataType::Binary, true),
696740
false,
697741
true,
698742
)]);
@@ -720,6 +764,104 @@ mod tests {
720764
);
721765
}
722766

767+
#[test]
768+
fn nested_coercion_leaves_binary_map_children_to_the_validating_cast() {
769+
use arrow::array::{
770+
Array, ArrayRef, AsArray, BinaryBuilder, MapBuilder, MapFieldNames,
771+
StringBuilder,
772+
};
773+
use arrow::record_batch::RecordBatch;
774+
use bytes::Bytes;
775+
use parquet::arrow::ArrowWriter;
776+
use parquet::arrow::arrow_reader::{
777+
ArrowReaderOptions, ParquetRecordBatchReaderBuilder,
778+
};
779+
use parquet::arrow::arrow_writer::ArrowWriterOptions;
780+
781+
// A binary map value holding bytes that are not valid UTF-8. Reading it
782+
// as a string would produce an invalid array, since the Parquet reader
783+
// only validates columns annotated `UTF8`, so the value must be left
784+
// binary for the (validating) cast to reject.
785+
let parquet_names = MapFieldNames {
786+
entry: "key_value".to_string(),
787+
key: "key".to_string(),
788+
value: "value".to_string(),
789+
};
790+
let mut builder = MapBuilder::new(
791+
Some(parquet_names),
792+
StringBuilder::new(),
793+
BinaryBuilder::new(),
794+
);
795+
builder.keys().append_value("k1");
796+
builder.values().append_value([0xff]);
797+
builder.append(true).unwrap();
798+
let batch =
799+
RecordBatch::try_from_iter([("m", Arc::new(builder.finish()) as ArrayRef)])
800+
.unwrap();
801+
802+
let mut bytes = Vec::new();
803+
let mut writer = ArrowWriter::try_new_with_options(
804+
&mut bytes,
805+
batch.schema(),
806+
ArrowWriterOptions::new().with_skip_arrow_metadata(true),
807+
)
808+
.unwrap();
809+
writer.write(&batch).unwrap();
810+
writer.close().unwrap();
811+
let bytes = Bytes::from(bytes);
812+
813+
let file_schema = ParquetRecordBatchReaderBuilder::try_new(bytes.clone())
814+
.unwrap()
815+
.schema()
816+
.clone();
817+
let table_schema = Schema::new(vec![Field::new_map(
818+
"m",
819+
"entries",
820+
Field::new("keys", DataType::Utf8View, false),
821+
Field::new("values", DataType::Utf8View, true),
822+
false,
823+
true,
824+
)]);
825+
826+
// The key is still coerced by position, the binary value is not
827+
let coerced =
828+
apply_file_schema_type_coercions(&table_schema, &file_schema).unwrap();
829+
assert_eq!(
830+
coerced.field(0).data_type(),
831+
&DataType::Map(
832+
Arc::new(Field::new_struct(
833+
"key_value",
834+
vec![
835+
Field::new("key", DataType::Utf8View, false),
836+
Field::new("value", DataType::Binary, true),
837+
],
838+
false,
839+
)),
840+
false,
841+
)
842+
);
843+
844+
let mut reader = ParquetRecordBatchReaderBuilder::try_new_with_options(
845+
bytes,
846+
ArrowReaderOptions::new().with_schema(Arc::new(coerced)),
847+
)
848+
.unwrap()
849+
.build()
850+
.unwrap();
851+
let decoded = reader.next().unwrap().unwrap();
852+
// The decoded batch is valid: the invalid bytes are still binary
853+
decoded.column(0).to_data().validate_full().unwrap();
854+
assert_eq!(
855+
decoded
856+
.column(0)
857+
.as_map()
858+
.values()
859+
.as_binary::<i32>()
860+
.value(0),
861+
[0xff]
862+
);
863+
}
864+
723865
#[test]
724866
fn nested_coercion_list_inside_struct() {
725867
let file_schema = Schema::new(vec![Field::new_struct(
@@ -760,8 +902,7 @@ mod tests {
760902
#[test]
761903
fn nested_coercion_map_reader_produces_string_views() {
762904
use arrow::array::{
763-
Array, ArrayRef, AsArray, BinaryBuilder, MapBuilder, MapFieldNames,
764-
StringBuilder,
905+
Array, ArrayRef, AsArray, MapBuilder, MapFieldNames, StringBuilder,
765906
};
766907
use arrow::record_batch::RecordBatch;
767908
use bytes::Bytes;
@@ -782,10 +923,10 @@ mod tests {
782923
let mut builder = MapBuilder::new(
783924
Some(parquet_names),
784925
StringBuilder::new(),
785-
BinaryBuilder::new(),
926+
StringBuilder::new(),
786927
);
787928
builder.keys().append_value("k1");
788-
builder.values().append_value(b"v1");
929+
builder.values().append_value("v1");
789930
builder.append(true).unwrap();
790931
builder.append(false).unwrap();
791932
let map = builder.finish();
@@ -823,7 +964,7 @@ mod tests {
823964
"key_value",
824965
vec![
825966
Field::new("key", DataType::Utf8, false),
826-
Field::new("value", DataType::Binary, true),
967+
Field::new("value", DataType::Utf8, true),
827968
],
828969
false,
829970
)),

0 commit comments

Comments
 (0)