Skip to content

Commit a856c00

Browse files
author
ebembi-crdb
committed
feat: implement DFExtensionType for remaining canonical Arrow extension types
Closes apache#21144 Implements DFExtensionType for all remaining canonical Arrow extension types so they are recognized and pretty-printed by the extension type registry: - Bool8: displays Int8 values as 'true'/'false' instead of raw integers - Json: uses default string formatter (values are already valid JSON) - Opaque: uses default formatter - FixedShapeTensor: uses default formatter, storage_type computed from value_type and list_size - VariableShapeTensor: uses default formatter, storage_type computed from value_type and dimensions - TimestampWithOffset: uses default formatter All six types are registered in MemoryExtensionTypeRegistry::new_with_canonical_extension_types() alongside the existing UUID registration.
1 parent b1f8043 commit a856c00

9 files changed

Lines changed: 862 additions & 3 deletions

File tree

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::error::_internal_err;
19+
use crate::types::extension::DFExtensionType;
20+
use arrow::array::{Array, Int8Array};
21+
use arrow::datatypes::DataType;
22+
use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};
23+
use std::fmt::Write;
24+
25+
/// Defines the extension type logic for the canonical `arrow.bool8` extension type.
26+
///
27+
/// Bool8 values are displayed as `true` or `false`, where `0` maps to `false` and
28+
/// any non-zero value maps to `true`.
29+
///
30+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
31+
impl DFExtensionType for arrow_schema::extension::Bool8 {
32+
fn storage_type(&self) -> DataType {
33+
DataType::Int8
34+
}
35+
36+
fn serialize_metadata(&self) -> Option<String> {
37+
// Bool8 metadata is an empty string per the Arrow spec.
38+
Some(String::new())
39+
}
40+
41+
fn create_array_formatter<'fmt>(
42+
&self,
43+
array: &'fmt dyn Array,
44+
options: &FormatOptions<'fmt>,
45+
) -> crate::Result<Option<ArrayFormatter<'fmt>>> {
46+
if array.data_type() != &DataType::Int8 {
47+
return _internal_err!("Wrong array type for Bool8");
48+
}
49+
50+
let display_index = Bool8ValueDisplayIndex {
51+
array: array.as_any().downcast_ref().unwrap(),
52+
null_str: options.null(),
53+
};
54+
Ok(Some(ArrayFormatter::new(
55+
Box::new(display_index),
56+
options.safe(),
57+
)))
58+
}
59+
}
60+
61+
/// Pretty printer for 8-bit Boolean values.
62+
///
63+
/// Displays `false` for zero values and `true` for any non-zero value.
64+
#[derive(Debug, Clone, Copy)]
65+
struct Bool8ValueDisplayIndex<'a> {
66+
array: &'a Int8Array,
67+
null_str: &'a str,
68+
}
69+
70+
impl DisplayIndex for Bool8ValueDisplayIndex<'_> {
71+
fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult {
72+
if self.array.is_null(idx) {
73+
write!(f, "{}", self.null_str)?;
74+
return Ok(());
75+
}
76+
77+
let value = self.array.value(idx);
78+
if value == 0 {
79+
write!(f, "false")?;
80+
} else {
81+
write!(f, "true")?;
82+
}
83+
Ok(())
84+
}
85+
}
86+
87+
#[cfg(test)]
88+
mod tests {
89+
use super::*;
90+
use crate::ScalarValue;
91+
92+
#[test]
93+
pub fn test_pretty_print_bool8_false() {
94+
let bool8 = ScalarValue::Int8(Some(0)).to_array_of_size(1).unwrap();
95+
96+
let extension_type = arrow_schema::extension::Bool8 {};
97+
let formatter = extension_type
98+
.create_array_formatter(bool8.as_ref(), &FormatOptions::default())
99+
.unwrap()
100+
.unwrap();
101+
102+
assert_eq!(formatter.value(0).to_string(), "false");
103+
}
104+
105+
#[test]
106+
pub fn test_pretty_print_bool8_true() {
107+
let bool8 = ScalarValue::Int8(Some(1)).to_array_of_size(1).unwrap();
108+
109+
let extension_type = arrow_schema::extension::Bool8 {};
110+
let formatter = extension_type
111+
.create_array_formatter(bool8.as_ref(), &FormatOptions::default())
112+
.unwrap()
113+
.unwrap();
114+
115+
assert_eq!(formatter.value(0).to_string(), "true");
116+
}
117+
118+
#[test]
119+
pub fn test_pretty_print_bool8_nonzero_is_true() {
120+
// Any non-zero value should display as "true"
121+
let bool8 = ScalarValue::Int8(Some(42)).to_array_of_size(1).unwrap();
122+
123+
let extension_type = arrow_schema::extension::Bool8 {};
124+
let formatter = extension_type
125+
.create_array_formatter(bool8.as_ref(), &FormatOptions::default())
126+
.unwrap()
127+
.unwrap();
128+
129+
assert_eq!(formatter.value(0).to_string(), "true");
130+
}
131+
}
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::types::extension::DFExtensionType;
19+
use arrow::datatypes::DataType;
20+
use arrow_schema::extension::ExtensionType;
21+
22+
/// Defines the extension type logic for the canonical `arrow.fixed_shape_tensor` extension type.
23+
///
24+
/// Fixed shape tensors are stored as `FixedSizeList` arrays; the default Arrow formatter
25+
/// is used for display.
26+
///
27+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
28+
impl DFExtensionType for arrow_schema::extension::FixedShapeTensor {
29+
fn storage_type(&self) -> DataType {
30+
DataType::new_fixed_size_list(
31+
self.value_type().clone(),
32+
i32::try_from(self.list_size()).expect("list size overflow"),
33+
false,
34+
)
35+
}
36+
37+
fn serialize_metadata(&self) -> Option<String> {
38+
<arrow_schema::extension::FixedShapeTensor as ExtensionType>::serialize_metadata(
39+
self,
40+
)
41+
}
42+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::types::extension::DFExtensionType;
19+
use arrow::datatypes::DataType;
20+
use arrow_schema::extension::ExtensionType;
21+
22+
/// Defines the extension type logic for the canonical `arrow.json` extension type.
23+
///
24+
/// JSON values are already stored as UTF-8 strings, so the default Arrow string
25+
/// formatter is used for display.
26+
///
27+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
28+
impl DFExtensionType for arrow_schema::extension::Json {
29+
fn storage_type(&self) -> DataType {
30+
// JSON can be stored as Utf8, LargeUtf8, or Utf8View; Utf8 is the most common default.
31+
DataType::Utf8
32+
}
33+
34+
fn serialize_metadata(&self) -> Option<String> {
35+
<arrow_schema::extension::Json as ExtensionType>::serialize_metadata(self)
36+
}
37+
}
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
mod bool8;
19+
mod fixed_shape_tensor;
20+
mod json;
21+
mod opaque;
22+
mod timestamp_with_offset;
23+
mod uuid;
24+
mod variable_shape_tensor;
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::types::extension::DFExtensionType;
19+
use arrow::datatypes::DataType;
20+
use arrow_schema::extension::ExtensionType;
21+
22+
/// Defines the extension type logic for the canonical `arrow.opaque` extension type.
23+
///
24+
/// Opaque represents a type received from an external system that cannot be interpreted.
25+
/// The default Arrow formatter is used for display.
26+
///
27+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
28+
impl DFExtensionType for arrow_schema::extension::Opaque {
29+
fn storage_type(&self) -> DataType {
30+
// Opaque supports any storage type; Null is recommended when there is no underlying data.
31+
DataType::Null
32+
}
33+
34+
fn serialize_metadata(&self) -> Option<String> {
35+
<arrow_schema::extension::Opaque as ExtensionType>::serialize_metadata(self)
36+
}
37+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::types::extension::DFExtensionType;
19+
use arrow::datatypes::DataType;
20+
21+
/// Defines the extension type logic for the canonical `arrow.timestamp_with_offset` extension type.
22+
///
23+
/// Timestamp with offset values are stored as `Struct` arrays containing a UTC timestamp
24+
/// and an offset in minutes. The default Arrow formatter is used for display.
25+
///
26+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
27+
impl DFExtensionType for arrow_schema::extension::TimestampWithOffset {
28+
fn storage_type(&self) -> DataType {
29+
// TimestampWithOffset stores no internal state to determine the timestamp precision.
30+
// The actual storage type depends on the time unit chosen by the producer.
31+
// Returning Null here is a placeholder; the actual DataType is validated at registration
32+
// time via ExtensionType::supports_data_type.
33+
DataType::Null
34+
}
35+
36+
fn serialize_metadata(&self) -> Option<String> {
37+
// TimestampWithOffset has no metadata.
38+
None
39+
}
40+
}
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
use crate::types::extension::DFExtensionType;
19+
use arrow::datatypes::{DataType, Field, Fields};
20+
use arrow_schema::extension::ExtensionType;
21+
22+
/// Defines the extension type logic for the canonical `arrow.variable_shape_tensor` extension type.
23+
///
24+
/// Variable shape tensors are stored as `Struct` arrays containing `data` (a list of elements)
25+
/// and `shape` (a fixed-size list of int32 dimensions). The default Arrow formatter is used
26+
/// for display.
27+
///
28+
/// See [`DFExtensionType`] for information on DataFusion's extension type mechanism.
29+
impl DFExtensionType for arrow_schema::extension::VariableShapeTensor {
30+
fn storage_type(&self) -> DataType {
31+
let dims = i32::try_from(self.dimensions()).expect("dimensions overflow");
32+
DataType::Struct(Fields::from_iter([
33+
Field::new_list(
34+
"data",
35+
Field::new_list_field(self.value_type().clone(), false),
36+
false,
37+
),
38+
Field::new(
39+
"shape",
40+
DataType::new_fixed_size_list(DataType::Int32, dims, false),
41+
false,
42+
),
43+
]))
44+
}
45+
46+
fn serialize_metadata(&self) -> Option<String> {
47+
<arrow_schema::extension::VariableShapeTensor as ExtensionType>::serialize_metadata(self)
48+
}
49+
}

0 commit comments

Comments
 (0)