Skip to content

Commit b62c35a

Browse files
committed
fix(parquet): apply view types to dictionary values
1 parent 62c05c1 commit b62c35a

1 file changed

Lines changed: 50 additions & 0 deletions

File tree

‎datafusion/datasource-parquet/src/schema_coercion.rs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,20 @@ pub fn transform_schema_to_view(schema: &Schema) -> Schema {
809809
DataType::Binary | DataType::LargeBinary => {
810810
field_with_new_type(field, DataType::BinaryView)
811811
}
812+
DataType::Dictionary(key_type, value_type) => match value_type.as_ref() {
813+
DataType::Utf8 | DataType::LargeUtf8 => field_with_new_type(
814+
field,
815+
DataType::Dictionary(key_type.clone(), Box::new(DataType::Utf8View)),
816+
),
817+
DataType::Binary | DataType::LargeBinary => field_with_new_type(
818+
field,
819+
DataType::Dictionary(
820+
key_type.clone(),
821+
Box::new(DataType::BinaryView),
822+
),
823+
),
824+
_ => Arc::clone(field),
825+
},
812826
_ => Arc::clone(field),
813827
})
814828
.collect();
@@ -2264,4 +2278,40 @@ mod tests {
22642278
// Plain Binary is converted as before
22652279
assert_eq!(result.field(4).data_type(), &DataType::Utf8);
22662280
}
2281+
2282+
#[test]
2283+
fn transform_schema_to_view_handles_dictionary_value_types() {
2284+
let schema = Schema::new(vec![
2285+
Field::new(
2286+
"a",
2287+
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
2288+
true,
2289+
),
2290+
Field::new(
2291+
"b",
2292+
DataType::Dictionary(
2293+
Box::new(DataType::Int32),
2294+
Box::new(DataType::Binary),
2295+
),
2296+
true,
2297+
),
2298+
]);
2299+
2300+
let result = transform_schema_to_view(&schema);
2301+
2302+
assert_eq!(
2303+
result.field(0).data_type(),
2304+
&DataType::Dictionary(
2305+
Box::new(DataType::Int32),
2306+
Box::new(DataType::Utf8View)
2307+
),
2308+
);
2309+
assert_eq!(
2310+
result.field(1).data_type(),
2311+
&DataType::Dictionary(
2312+
Box::new(DataType::Int32),
2313+
Box::new(DataType::BinaryView)
2314+
),
2315+
);
2316+
}
22672317
}

0 commit comments

Comments
 (0)