Skip to content

Commit ca0043b

Browse files
committed
feat: add PostgreSQL-style ENUM types
Support `CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')` and ENUM columns, literals and casts: - `DFEnum` (`datafusion.enum` extension type) stores values as `Int16` codes and the type name and labels in the field metadata, so ENUM types survive a round trip through Parquet; - the type coercion analyzer converts labels to codes at planning time, so comparisons and sorting use the declaration order and Parquet statistics still prune data. It handles comparisons, `IN`, `BETWEEN`, `CAST` and `TRY_CAST`, and rejects arithmetic, comparisons of different ENUM types and casts to numbers; - unknown labels give the PostgreSQL error `invalid input value for enum mood: "angry"`; - `DataFrame::to_string`, datafusion-cli and sqllogictest show labels, not codes. Known limitations: ENUM metadata is lost in `CASE` / `COALESCE`, `MIN` / `MAX` and `UNION` (apache#24860), and ENUM columns are not supported in CSV external tables.
1 parent db8fe63 commit ca0043b

12 files changed

Lines changed: 978 additions & 14 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-cli/src/print_format.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
//! Print format variants
1919
2020
use std::str::FromStr;
21+
use std::sync::Arc;
2122

2223
use crate::print_options::MaxRows;
2324

@@ -28,6 +29,9 @@ use arrow::record_batch::RecordBatch;
2829
use arrow::util::pretty::pretty_format_batches_with_options;
2930
use datafusion::config::FormatOptions;
3031
use datafusion::error::Result;
32+
use datafusion::execution::SessionStateDefaults;
33+
use datafusion::logical_expr::extension_types::DFArrayFormatterFactory;
34+
use datafusion::logical_expr::registry::MemoryExtensionTypeRegistry;
3135

3236
/// Allow records to be printed in different formats
3337
#[derive(Debug, PartialEq, Eq, clap::ValueEnum, Clone, Copy)]
@@ -113,6 +117,12 @@ fn format_batches_with_maxrows<W: std::io::Write>(
113117
format_options: &FormatOptions,
114118
) -> Result<()> {
115119
let options: arrow::util::display::FormatOptions = format_options.try_into()?;
120+
// Format extension types such as enums with their default formatters.
121+
let registry = MemoryExtensionTypeRegistry::new_with_types(
122+
SessionStateDefaults::default_extension_types(),
123+
)?;
124+
let formatter_factory = DFArrayFormatterFactory::new(Arc::new(registry));
125+
let options = options.with_formatter_factory(Some(&formatter_factory));
116126

117127
match maxrows {
118128
MaxRows::Limited(maxrows) => {

‎datafusion/common/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ num-traits = { workspace = true }
8585
object_store = { workspace = true, optional = true }
8686
parquet = { workspace = true, optional = true, default-features = true }
8787
recursive = { workspace = true, optional = true }
88+
serde_json = { workspace = true }
8889
sqlparser = { workspace = true, optional = true }
8990
tokio = { workspace = true }
9091
uuid = { workspace = true, features = ["v4"] }
Lines changed: 316 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,316 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! PostgreSQL-style enumerated types.
19+
20+
use crate::Result;
21+
use crate::datatype::DataTypeExt;
22+
use crate::error::{_internal_err, _plan_datafusion_err, _plan_err};
23+
use crate::types::extension::DFExtensionType;
24+
use arrow::array::{Array, AsArray, Int16Array, StringArray};
25+
use arrow::datatypes::{DataType, Field, FieldRef};
26+
use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};
27+
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
28+
use std::collections::HashMap;
29+
use std::fmt::Write;
30+
use std::sync::Arc;
31+
32+
/// An enumerated type: a static, ordered set of text labels, like
33+
/// `CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')` in PostgreSQL.
34+
///
35+
/// Values are stored as [`DataType::Int16`] codes, where a code is the position of the label
36+
/// in the declaration. Comparing and sorting codes therefore follows the declaration order of
37+
/// the labels, as in PostgreSQL.
38+
///
39+
/// The type is identified by the `datafusion.enum` extension type name. Its metadata is a JSON
40+
/// object with the type name and the labels, for example
41+
/// `{"name":"mood","labels":["sad","ok","happy"]}`. As the metadata is part of the Arrow schema,
42+
/// file formats that store the Arrow schema (such as Parquet and Arrow IPC) keep the type
43+
/// definition next to the data.
44+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
45+
pub struct DFEnum {
46+
name: String,
47+
labels: Vec<String>,
48+
}
49+
50+
impl DFEnum {
51+
/// The extension type name of enumerated types.
52+
pub const NAME: &'static str = "datafusion.enum";
53+
54+
/// The storage type of the codes.
55+
pub const STORAGE_TYPE: DataType = DataType::Int16;
56+
57+
/// Creates a new enumerated type named `name` with the `labels` in declaration order.
58+
pub fn try_new(name: impl Into<String>, labels: Vec<String>) -> Result<Self> {
59+
let name = name.into();
60+
if labels.len() > i16::MAX as usize + 1 {
61+
return _plan_err!(
62+
"Enum type {name} has {} labels, at most {} are supported",
63+
labels.len(),
64+
i16::MAX as usize + 1
65+
);
66+
}
67+
let mut seen = std::collections::HashSet::with_capacity(labels.len());
68+
for label in &labels {
69+
if !seen.insert(label.as_str()) {
70+
return _plan_err!("Enum label \"{label}\" used more than once");
71+
}
72+
}
73+
Ok(Self { name, labels })
74+
}
75+
76+
/// Creates an enumerated type from the storage type and the serialized metadata of a field.
77+
pub fn try_from_metadata(
78+
data_type: &DataType,
79+
metadata: Option<&str>,
80+
) -> Result<Self> {
81+
if data_type != &Self::STORAGE_TYPE {
82+
return _plan_err!(
83+
"Enum type requires storage type {}, got {data_type}",
84+
Self::STORAGE_TYPE
85+
);
86+
}
87+
let Some(metadata) = metadata else {
88+
return _plan_err!("Enum type requires metadata");
89+
};
90+
let invalid = || _plan_datafusion_err!("Invalid enum type metadata: {metadata}");
91+
let value: serde_json::Value =
92+
serde_json::from_str(metadata).map_err(|_| invalid())?;
93+
let name = value
94+
.get("name")
95+
.and_then(|name| name.as_str())
96+
.ok_or_else(invalid)?;
97+
let labels = value
98+
.get("labels")
99+
.and_then(|labels| labels.as_array())
100+
.ok_or_else(invalid)?
101+
.iter()
102+
.map(|label| label.as_str().map(str::to_owned).ok_or_else(invalid))
103+
.collect::<Result<Vec<_>>>()?;
104+
Self::try_new(name, labels)
105+
}
106+
107+
/// Returns the enumerated type of `field`, or `None` if the field is not an enum.
108+
pub fn try_from_field(field: &Field) -> Result<Option<Self>> {
109+
if field.extension_type_name() != Some(Self::NAME) {
110+
return Ok(None);
111+
}
112+
Self::try_from_metadata(field.data_type(), field.extension_type_metadata())
113+
.map(Some)
114+
}
115+
116+
/// The name of the type.
117+
pub fn name(&self) -> &str {
118+
&self.name
119+
}
120+
121+
/// The labels in declaration order.
122+
pub fn labels(&self) -> &[String] {
123+
&self.labels
124+
}
125+
126+
/// Returns the metadata to attach to a field of this type.
127+
pub fn field_metadata(&self) -> HashMap<String, String> {
128+
HashMap::from([
129+
(EXTENSION_TYPE_NAME_KEY.to_owned(), Self::NAME.to_owned()),
130+
(
131+
EXTENSION_TYPE_METADATA_KEY.to_owned(),
132+
self.serialize_metadata_json(),
133+
),
134+
])
135+
}
136+
137+
/// Returns a nullable, unnamed field of this type.
138+
pub fn to_field_ref(&self) -> FieldRef {
139+
Arc::new(
140+
Self::STORAGE_TYPE
141+
.into_nullable_field()
142+
.with_metadata(self.field_metadata()),
143+
)
144+
}
145+
146+
/// Returns the code of `label`.
147+
pub fn code(&self, label: &str) -> Result<i16> {
148+
self.labels
149+
.iter()
150+
.position(|l| l == label)
151+
.map(|code| code as i16)
152+
.ok_or_else(|| {
153+
_plan_datafusion_err!(
154+
"invalid input value for enum {}: \"{label}\"",
155+
self.name
156+
)
157+
})
158+
}
159+
160+
/// Returns the label of `code`.
161+
pub fn label(&self, code: i16) -> Result<&str> {
162+
usize::try_from(code)
163+
.ok()
164+
.and_then(|code| self.labels.get(code))
165+
.map(String::as_str)
166+
.ok_or_else(|| {
167+
_plan_datafusion_err!("invalid code for enum {}: {code}", self.name)
168+
})
169+
}
170+
171+
/// Converts an array of labels (`Utf8`, `LargeUtf8` or `Utf8View`) to an array of codes.
172+
pub fn encode(&self, array: &dyn Array) -> Result<Int16Array> {
173+
let codes: HashMap<&str, i16> = self
174+
.labels
175+
.iter()
176+
.enumerate()
177+
.map(|(code, label)| (label.as_str(), code as i16))
178+
.collect();
179+
let encode = |label: Option<&str>| {
180+
label
181+
.map(|label| {
182+
codes.get(label).copied().ok_or_else(|| {
183+
_plan_datafusion_err!(
184+
"invalid input value for enum {}: \"{label}\"",
185+
self.name
186+
)
187+
})
188+
})
189+
.transpose()
190+
};
191+
match array.data_type() {
192+
DataType::Utf8 => array.as_string::<i32>().iter().map(encode).collect(),
193+
DataType::LargeUtf8 => array.as_string::<i64>().iter().map(encode).collect(),
194+
DataType::Utf8View => array.as_string_view().iter().map(encode).collect(),
195+
other => _internal_err!("Cannot convert {other} to enum {}", self.name),
196+
}
197+
}
198+
199+
/// Converts an array of codes to an array of labels.
200+
pub fn decode(&self, array: &Int16Array) -> Result<StringArray> {
201+
array
202+
.iter()
203+
.map(|code| code.map(|code| self.label(code)).transpose())
204+
.collect()
205+
}
206+
207+
fn serialize_metadata_json(&self) -> String {
208+
serde_json::json!({ "name": self.name, "labels": self.labels }).to_string()
209+
}
210+
}
211+
212+
impl DFExtensionType for DFEnum {
213+
fn storage_type(&self) -> DataType {
214+
Self::STORAGE_TYPE
215+
}
216+
217+
fn serialize_metadata(&self) -> Option<String> {
218+
Some(self.serialize_metadata_json())
219+
}
220+
221+
fn create_array_formatter<'fmt>(
222+
&self,
223+
array: &'fmt dyn Array,
224+
options: &FormatOptions<'fmt>,
225+
) -> Result<Option<ArrayFormatter<'fmt>>> {
226+
let Some(codes) = array.as_primitive_opt() else {
227+
return _internal_err!("Wrong array type for enum {}", self.name);
228+
};
229+
let display_index = EnumValueDisplayIndex {
230+
codes,
231+
labels: self.labels.clone(),
232+
null_str: options.null(),
233+
};
234+
Ok(Some(ArrayFormatter::new(
235+
Box::new(display_index),
236+
options.safe(),
237+
)))
238+
}
239+
}
240+
241+
/// Pretty printer for enum codes.
242+
struct EnumValueDisplayIndex<'a> {
243+
codes: &'a Int16Array,
244+
labels: Vec<String>,
245+
null_str: &'a str,
246+
}
247+
248+
impl DisplayIndex for EnumValueDisplayIndex<'_> {
249+
fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult {
250+
if self.codes.is_null(idx) {
251+
write!(f, "{}", self.null_str)?;
252+
return Ok(());
253+
}
254+
let code = self.codes.value(idx);
255+
match usize::try_from(code).ok().and_then(|c| self.labels.get(c)) {
256+
Some(label) => write!(f, "{label}")?,
257+
None => write!(f, "<invalid enum code {code}>")?,
258+
}
259+
Ok(())
260+
}
261+
}
262+
263+
#[cfg(test)]
264+
mod tests {
265+
use super::*;
266+
267+
fn mood() -> DFEnum {
268+
DFEnum::try_new("mood", vec!["sad".into(), "ok".into(), "happy".into()]).unwrap()
269+
}
270+
271+
#[test]
272+
fn metadata_roundtrip() -> Result<()> {
273+
let mood = mood();
274+
let field = mood.to_field_ref();
275+
assert_eq!(DFEnum::try_from_field(&field)?, Some(mood));
276+
assert_eq!(
277+
DFEnum::try_from_field(&Field::new("", DataType::Int16, true))?,
278+
None
279+
);
280+
Ok(())
281+
}
282+
283+
#[test]
284+
fn rejects_duplicate_labels() {
285+
let err = DFEnum::try_new("e", vec!["a".into(), "a".into()]).unwrap_err();
286+
assert!(err.to_string().contains("used more than once"), "{err}");
287+
}
288+
289+
#[test]
290+
fn encode_decode() -> Result<()> {
291+
let mood = mood();
292+
let labels = StringArray::from(vec![Some("happy"), None, Some("sad")]);
293+
let codes = mood.encode(&labels)?;
294+
assert_eq!(codes, Int16Array::from(vec![Some(2), None, Some(0)]));
295+
assert_eq!(mood.decode(&codes)?, labels);
296+
297+
let err = mood.encode(&StringArray::from(vec!["angry"])).unwrap_err();
298+
assert!(
299+
err.to_string()
300+
.contains("invalid input value for enum mood: \"angry\""),
301+
"{err}"
302+
);
303+
Ok(())
304+
}
305+
306+
#[test]
307+
fn pretty_print() -> Result<()> {
308+
let codes = Int16Array::from(vec![Some(1), None]);
309+
let formatter = mood()
310+
.create_array_formatter(&codes, &FormatOptions::default().with_null("NULL"))?
311+
.unwrap();
312+
assert_eq!(formatter.value(0).to_string(), "ok");
313+
assert_eq!(formatter.value(1).to_string(), "NULL");
314+
Ok(())
315+
}
316+
}

‎datafusion/common/src/types/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717

1818
mod builtin;
1919
mod canonical_extensions;
20+
mod enum_type;
2021
mod extension;
2122
mod field;
2223
mod logical;
2324
mod native;
2425

2526
pub use builtin::*;
2627
pub use canonical_extensions::*;
28+
pub use enum_type::*;
2729
pub use extension::*;
2830
pub use field::*;
2931
pub use logical::*;

0 commit comments

Comments
 (0)