You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit fd1c5b3
Browse filesBrowse the repository at this point in the historyBrowse files
Safely ignore Parquet fields with unimplemented Thrift types (#9974)
# Which issue does this PR close?
- Closes#9973.
# Rationale for this change
The thrift decoder should be able to skip fields with unimplemented
thrift types `set`, `map`, and `uuid`.
# What changes are included in this PR?
Flesh out the thrift enums and add code to the skip function to handle
the above types.
# Are these changes tested?
Yes, test is added
# Are there any user-facing changes?
No, changes are to internal APIs
// see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#struct
502
508
FieldType::Struct => {
503
-
letmut last_field_id = 0i16;
504
509
loop{
505
-
let field_ident = self.read_field_begin(last_field_id)?;
510
+
// we don't need field id for skipping, so always pass 0 for last id
// see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#list-and-set
521
+
FieldType::List | FieldType::Set => {
515
522
let list_ident = self.read_list_begin()?;
523
+
let element_type = FieldType::from(list_ident.element_type);
516
524
for _ in0..list_ident.size{
517
-
let element_type = FieldType::try_from(list_ident.element_type)?;
518
525
self.skip_till_depth(element_type, depth - 1)?;
519
526
}
520
527
Ok(())
521
528
}
522
-
// no list or map types in parquet format
529
+
// see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#map
530
+
FieldType::Map => {
531
+
let size = i32::try_from(self.read_vlq()?)?;
532
+
if size > 0{
533
+
let kv = self.read_byte()?;
534
+
let key_type = FieldType::from(ElementType::try_from(kv >> 4)?);
535
+
let val_type = FieldType::from(ElementType::try_from(kv &0xf)?);
536
+
for _ in0..size {
537
+
self.skip_till_depth(key_type, depth - 1)?;
538
+
self.skip_till_depth(val_type, depth - 1)?;
539
+
}
540
+
}
541
+
Ok(())
542
+
}
543
+
// see https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md#universal-unique-identifier-encoding
0 commit comments