2222//! These helpers are independent of the [`ParquetFormat`](crate::file_format::ParquetFormat)
2323//! type and several have been re-exported at the crate root for use by
2424//! callers outside the format implementation.
25+ //!
26+ //! # Binary to string coercion
27+ //!
28+ //! Coercing a binary file column to a string type makes the Parquet reader
29+ //! build the string array itself. It only validates UTF-8 for columns carrying
30+ //! the `UTF8` logical annotation, which such a column by definition does not,
31+ //! so invalid bytes surface as an invalid string array rather than an error.
32+ //! That is long-standing behaviour for top-level and struct fields; it is not
33+ //! extended to map children, whose positional matching is newer, so binary map
34+ //! values keep going through the validating cast instead.
2535
2636use std:: cell:: RefCell ;
2737use std:: collections:: { HashMap , HashSet } ;
@@ -66,7 +76,8 @@ pub fn apply_file_schema_type_coercions(
6676 table_schema : & Schema ,
6777 file_schema : & Schema ,
6878) -> Option < Schema > {
69- let fields = coerce_fields_by_name ( table_schema. fields ( ) , file_schema. fields ( ) ) ?;
79+ let fields =
80+ coerce_fields_by_name ( table_schema. fields ( ) , file_schema. fields ( ) , true ) ?;
7081 Some ( Schema :: new_with_metadata (
7182 fields,
7283 file_schema. metadata . clone ( ) ,
@@ -78,7 +89,11 @@ pub fn apply_file_schema_type_coercions(
7889/// File fields with no counterpart in `table_fields` are kept unchanged and
7990/// table fields missing from the file are ignored. Returns `None` if no field
8091/// changed.
81- fn coerce_fields_by_name ( table_fields : & Fields , file_fields : & Fields ) -> Option < Fields > {
92+ fn coerce_fields_by_name (
93+ table_fields : & Fields ,
94+ file_fields : & Fields ,
95+ binary_to_string : bool ,
96+ ) -> Option < Fields > {
8297 // Create a mapping of table field names to their data types for fast lookup
8398 let table_types: HashMap < _ , _ > = table_fields
8499 . iter ( )
@@ -87,7 +102,7 @@ fn coerce_fields_by_name(table_fields: &Fields, file_fields: &Fields) -> Option<
87102
88103 coerce_fields ( file_fields, |_, field| {
89104 let table_type = table_types. get ( field. name ( ) ) ?;
90- coerce_data_type ( table_type, field. data_type ( ) )
105+ coerce_data_type ( table_type, field. data_type ( ) , binary_to_string )
91106 . map ( |new_type| field_with_new_type ( field, new_type) )
92107 } )
93108}
@@ -127,40 +142,53 @@ fn coerce_fields(
127142
128143/// Coerce `file_type` towards `table_type`, recursing into nested types.
129144///
145+ /// `binary_to_string` allows binary file types to be read as string types.
146+ /// Parquet only validates UTF-8 when the column carries the `UTF8` logical
147+ /// annotation, which a binary column by definition does not, so such a
148+ /// conversion hands back an unvalidated string array; see the module docs.
149+ ///
130150/// Returns the new type for the file field, or `None` if no transformation
131151/// is needed (including when the two types are unrelated).
132- fn coerce_data_type ( table_type : & DataType , file_type : & DataType ) -> Option < DataType > {
152+ fn coerce_data_type (
153+ table_type : & DataType ,
154+ file_type : & DataType ,
155+ binary_to_string : bool ,
156+ ) -> Option < DataType > {
133157 use DataType :: * ;
134158 match ( table_type, file_type) {
135159 // table schema uses string type, coerce the file schema to use string type
136- ( Utf8 , Binary | LargeBinary | BinaryView ) => Some ( Utf8 ) ,
160+ ( Utf8 , Binary | LargeBinary | BinaryView ) if binary_to_string => Some ( Utf8 ) ,
137161 // table schema uses large string type, coerce the file schema to use large string type
138- ( LargeUtf8 , Binary | LargeBinary | BinaryView ) => Some ( LargeUtf8 ) ,
162+ ( LargeUtf8 , Binary | LargeBinary | BinaryView ) if binary_to_string => {
163+ Some ( LargeUtf8 )
164+ }
139165 // table schema uses string view type, coerce the file schema to use view type
140- ( Utf8View , Binary | LargeBinary | BinaryView | Utf8 | LargeUtf8 ) => {
166+ ( Utf8View , Binary | LargeBinary | BinaryView ) if binary_to_string => {
141167 Some ( Utf8View )
142168 }
169+ ( Utf8View , Utf8 | LargeUtf8 ) => Some ( Utf8View ) ,
143170 ( BinaryView , Binary | LargeBinary ) => Some ( BinaryView ) ,
144171 // Struct children match by name
145172 ( Struct ( table_fields) , Struct ( file_fields) ) => {
146- coerce_fields_by_name ( table_fields, file_fields) . map ( Struct )
173+ coerce_fields_by_name ( table_fields, file_fields, binary_to_string ) . map ( Struct )
147174 }
148175 // List-like children match by position, regardless of their names.
149176 // The container kind and FixedSizeList width always come from the file.
150177 ( List ( table_child) , List ( file_child) ) => {
151- coerce_child ( table_child, file_child) . map ( List )
178+ coerce_child ( table_child, file_child, binary_to_string ) . map ( List )
152179 }
153180 ( LargeList ( table_child) , LargeList ( file_child) ) => {
154- coerce_child ( table_child, file_child) . map ( LargeList )
181+ coerce_child ( table_child, file_child, binary_to_string ) . map ( LargeList )
155182 }
156183 ( ListView ( table_child) , ListView ( file_child) ) => {
157- coerce_child ( table_child, file_child) . map ( ListView )
184+ coerce_child ( table_child, file_child, binary_to_string ) . map ( ListView )
158185 }
159186 ( LargeListView ( table_child) , LargeListView ( file_child) ) => {
160- coerce_child ( table_child, file_child) . map ( LargeListView )
187+ coerce_child ( table_child, file_child, binary_to_string ) . map ( LargeListView )
161188 }
162189 ( FixedSizeList ( table_child, _) , FixedSizeList ( file_child, size) ) => {
163- coerce_child ( table_child, file_child) . map ( |child| FixedSizeList ( child, * size) )
190+ coerce_child ( table_child, file_child, binary_to_string)
191+ . map ( |child| FixedSizeList ( child, * size) )
164192 }
165193 // Map keys and values match by position: Parquet always names them
166194 // `key`/`value` while Arrow producers commonly use `keys`/`values`.
@@ -174,13 +202,26 @@ fn coerce_data_type(table_type: &DataType, file_type: &DataType) -> Option<DataT
174202
175203/// Coerce a single nested child field, keeping everything but its data type
176204/// from `file_child`.
177- fn coerce_child ( table_child : & FieldRef , file_child : & FieldRef ) -> Option < FieldRef > {
178- coerce_data_type ( table_child. data_type ( ) , file_child. data_type ( ) )
179- . map ( |new_type| field_with_new_type ( file_child, new_type) )
205+ fn coerce_child (
206+ table_child : & FieldRef ,
207+ file_child : & FieldRef ,
208+ binary_to_string : bool ,
209+ ) -> Option < FieldRef > {
210+ coerce_data_type (
211+ table_child. data_type ( ) ,
212+ file_child. data_type ( ) ,
213+ binary_to_string,
214+ )
215+ . map ( |new_type| field_with_new_type ( file_child, new_type) )
180216}
181217
182218/// Coerce the `entries` struct of a [`DataType::Map`], matching the key and
183219/// value children by position.
220+ ///
221+ /// Binary children are never read as strings here: matching by position newly
222+ /// reaches map children whose names differ, and turning those into unvalidated
223+ /// string arrays would be a regression over leaving them to the (validating)
224+ /// cast. See the module docs.
184225fn coerce_map_entries (
185226 table_entries : & FieldRef ,
186227 file_entries : & FieldRef ,
@@ -195,7 +236,7 @@ fn coerce_map_entries(
195236 }
196237
197238 let fields = coerce_fields ( file_fields, |idx, file_child| {
198- coerce_child ( & table_fields[ idx] , file_child)
239+ coerce_child ( & table_fields[ idx] , file_child, false )
199240 } ) ?;
200241
201242 Some ( field_with_new_type ( file_entries, DataType :: Struct ( fields) ) )
@@ -629,7 +670,7 @@ mod tests {
629670 "key_value" ,
630671 vec ! [
631672 Field :: new( "key" , DataType :: Utf8 , false ) ,
632- Field :: new( "value" , DataType :: Binary , true ) ,
673+ Field :: new( "value" , DataType :: Utf8 , true ) ,
633674 ] ,
634675 false ,
635676 )
@@ -684,15 +725,18 @@ mod tests {
684725 "m" ,
685726 "entries" ,
686727 Field :: new( "keys" , DataType :: Utf8View , false ) ,
687- Field :: new( "values" , DataType :: Utf8 , true ) ,
728+ Field :: new( "values" , DataType :: Utf8View , true ) ,
688729 false ,
689730 true ,
690731 ) ] ) ;
732+ // The key is matched by position and coerced to a view, while the
733+ // binary value is left to the validating cast (see
734+ // `coerce_map_entries`)
691735 let expected = Schema :: new ( vec ! [ Field :: new_map(
692736 "m" ,
693737 "key_value" ,
694738 Field :: new( "key" , DataType :: Utf8View , false ) ,
695- Field :: new( "value" , DataType :: Utf8 , true ) ,
739+ Field :: new( "value" , DataType :: Binary , true ) ,
696740 false ,
697741 true ,
698742 ) ] ) ;
@@ -720,6 +764,104 @@ mod tests {
720764 ) ;
721765 }
722766
767+ #[ test]
768+ fn nested_coercion_leaves_binary_map_children_to_the_validating_cast ( ) {
769+ use arrow:: array:: {
770+ Array , ArrayRef , AsArray , BinaryBuilder , MapBuilder , MapFieldNames ,
771+ StringBuilder ,
772+ } ;
773+ use arrow:: record_batch:: RecordBatch ;
774+ use bytes:: Bytes ;
775+ use parquet:: arrow:: ArrowWriter ;
776+ use parquet:: arrow:: arrow_reader:: {
777+ ArrowReaderOptions , ParquetRecordBatchReaderBuilder ,
778+ } ;
779+ use parquet:: arrow:: arrow_writer:: ArrowWriterOptions ;
780+
781+ // A binary map value holding bytes that are not valid UTF-8. Reading it
782+ // as a string would produce an invalid array, since the Parquet reader
783+ // only validates columns annotated `UTF8`, so the value must be left
784+ // binary for the (validating) cast to reject.
785+ let parquet_names = MapFieldNames {
786+ entry : "key_value" . to_string ( ) ,
787+ key : "key" . to_string ( ) ,
788+ value : "value" . to_string ( ) ,
789+ } ;
790+ let mut builder = MapBuilder :: new (
791+ Some ( parquet_names) ,
792+ StringBuilder :: new ( ) ,
793+ BinaryBuilder :: new ( ) ,
794+ ) ;
795+ builder. keys ( ) . append_value ( "k1" ) ;
796+ builder. values ( ) . append_value ( [ 0xff ] ) ;
797+ builder. append ( true ) . unwrap ( ) ;
798+ let batch =
799+ RecordBatch :: try_from_iter ( [ ( "m" , Arc :: new ( builder. finish ( ) ) as ArrayRef ) ] )
800+ . unwrap ( ) ;
801+
802+ let mut bytes = Vec :: new ( ) ;
803+ let mut writer = ArrowWriter :: try_new_with_options (
804+ & mut bytes,
805+ batch. schema ( ) ,
806+ ArrowWriterOptions :: new ( ) . with_skip_arrow_metadata ( true ) ,
807+ )
808+ . unwrap ( ) ;
809+ writer. write ( & batch) . unwrap ( ) ;
810+ writer. close ( ) . unwrap ( ) ;
811+ let bytes = Bytes :: from ( bytes) ;
812+
813+ let file_schema = ParquetRecordBatchReaderBuilder :: try_new ( bytes. clone ( ) )
814+ . unwrap ( )
815+ . schema ( )
816+ . clone ( ) ;
817+ let table_schema = Schema :: new ( vec ! [ Field :: new_map(
818+ "m" ,
819+ "entries" ,
820+ Field :: new( "keys" , DataType :: Utf8View , false ) ,
821+ Field :: new( "values" , DataType :: Utf8View , true ) ,
822+ false ,
823+ true ,
824+ ) ] ) ;
825+
826+ // The key is still coerced by position, the binary value is not
827+ let coerced =
828+ apply_file_schema_type_coercions ( & table_schema, & file_schema) . unwrap ( ) ;
829+ assert_eq ! (
830+ coerced. field( 0 ) . data_type( ) ,
831+ & DataType :: Map (
832+ Arc :: new( Field :: new_struct(
833+ "key_value" ,
834+ vec![
835+ Field :: new( "key" , DataType :: Utf8View , false ) ,
836+ Field :: new( "value" , DataType :: Binary , true ) ,
837+ ] ,
838+ false ,
839+ ) ) ,
840+ false ,
841+ )
842+ ) ;
843+
844+ let mut reader = ParquetRecordBatchReaderBuilder :: try_new_with_options (
845+ bytes,
846+ ArrowReaderOptions :: new ( ) . with_schema ( Arc :: new ( coerced) ) ,
847+ )
848+ . unwrap ( )
849+ . build ( )
850+ . unwrap ( ) ;
851+ let decoded = reader. next ( ) . unwrap ( ) . unwrap ( ) ;
852+ // The decoded batch is valid: the invalid bytes are still binary
853+ decoded. column ( 0 ) . to_data ( ) . validate_full ( ) . unwrap ( ) ;
854+ assert_eq ! (
855+ decoded
856+ . column( 0 )
857+ . as_map( )
858+ . values( )
859+ . as_binary:: <i32 >( )
860+ . value( 0 ) ,
861+ [ 0xff ]
862+ ) ;
863+ }
864+
723865 #[ test]
724866 fn nested_coercion_list_inside_struct ( ) {
725867 let file_schema = Schema :: new ( vec ! [ Field :: new_struct(
@@ -760,8 +902,7 @@ mod tests {
760902 #[ test]
761903 fn nested_coercion_map_reader_produces_string_views ( ) {
762904 use arrow:: array:: {
763- Array , ArrayRef , AsArray , BinaryBuilder , MapBuilder , MapFieldNames ,
764- StringBuilder ,
905+ Array , ArrayRef , AsArray , MapBuilder , MapFieldNames , StringBuilder ,
765906 } ;
766907 use arrow:: record_batch:: RecordBatch ;
767908 use bytes:: Bytes ;
@@ -782,10 +923,10 @@ mod tests {
782923 let mut builder = MapBuilder :: new (
783924 Some ( parquet_names) ,
784925 StringBuilder :: new ( ) ,
785- BinaryBuilder :: new ( ) ,
926+ StringBuilder :: new ( ) ,
786927 ) ;
787928 builder. keys ( ) . append_value ( "k1" ) ;
788- builder. values ( ) . append_value ( b "v1") ;
929+ builder. values ( ) . append_value ( "v1" ) ;
789930 builder. append ( true ) . unwrap ( ) ;
790931 builder. append ( false ) . unwrap ( ) ;
791932 let map = builder. finish ( ) ;
@@ -823,7 +964,7 @@ mod tests {
823964 "key_value" ,
824965 vec![
825966 Field :: new( "key" , DataType :: Utf8 , false ) ,
826- Field :: new( "value" , DataType :: Binary , true ) ,
967+ Field :: new( "value" , DataType :: Utf8 , true ) ,
827968 ] ,
828969 false ,
829970 ) ) ,
0 commit comments