Skip to content

Commit 42bf024

Browse files
committed
feat: add PostgreSQL-style ENUM types
Support `CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')` and ENUM columns, literals and casts: - `DFEnum` (`datafusion.enum` extension type) stores values as `Int16` codes and the type name and labels in the field metadata, so ENUM types survive a round trip through Parquet; - the type coercion analyzer converts labels to codes at planning time, so comparisons and sorting use the declaration order and Parquet statistics still prune data. It handles comparisons, `IN`, `BETWEEN`, `CAST` and `TRY_CAST`, and rejects arithmetic, comparisons of different ENUM types and casts to numbers; - unknown labels give the PostgreSQL error `invalid input value for enum mood: "angry"`; - `DataFrame::to_string`, datafusion-cli and sqllogictest show labels, not codes. Known limitations: ENUM metadata is lost in `CASE` / `COALESCE`, `MIN` / `MAX` and `UNION` (apache#24860), and ENUM columns are not supported in CSV external tables.
1 parent e2fc9ae commit 42bf024

12 files changed

Lines changed: 937 additions & 12 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-cli/src/print_format.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
//! Print format variants
66
77
use std::str::FromStr;
8+
use std::sync::Arc;
89

910
use crate::print_options::MaxRows;
1011

@@ -15,6 +16,9 @@ use arrow::record_batch::RecordBatch;
1516
use arrow::util::pretty::pretty_format_batches_with_options;
1617
use datafusion::config::FormatOptions;
1718
use datafusion::error::Result;
19+
use datafusion::execution::SessionStateDefaults;
20+
use datafusion::logical_expr::extension_types::DFArrayFormatterFactory;
21+
use datafusion::logical_expr::registry::MemoryExtensionTypeRegistry;
1822

1923
/// Allow records to be printed in different formats
2024
#[derive(Debug, PartialEq, Eq, clap::ValueEnum, Clone, Copy)]
@@ -100,6 +104,12 @@ fn format_batches_with_maxrows<W: std::io::Write>(
100104
format_options: &FormatOptions,
101105
) -> Result<()> {
102106
let options: arrow::util::display::FormatOptions = format_options.try_into()?;
107+
// Format extension types such as enums with their default formatters.
108+
let registry = MemoryExtensionTypeRegistry::new_with_types(
109+
SessionStateDefaults::default_extension_types(),
110+
)?;
111+
let formatter_factory = DFArrayFormatterFactory::new(Arc::new(registry));
112+
let options = options.with_formatter_factory(Some(&formatter_factory));
103113

104114
match maxrows {
105115
MaxRows::Limited(maxrows) => {

‎datafusion/common/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ num-traits = { workspace = true }
7272
object_store = { workspace = true, optional = true }
7373
parquet = { workspace = true, optional = true, default-features = true }
7474
recursive = { workspace = true, optional = true }
75+
serde_json = { workspace = true }
7576
sqlparser = { workspace = true, optional = true }
7677
tokio = { workspace = true }
7778
uuid = { workspace = true, features = ["v4"] }
Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3+
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4+
5+
//! PostgreSQL-style enumerated types.
6+
7+
use crate::Result;
8+
use crate::datatype::DataTypeExt;
9+
use crate::error::{_internal_err, _plan_datafusion_err, _plan_err};
10+
use crate::types::extension::DFExtensionType;
11+
use arrow::array::{Array, AsArray, Int16Array, StringArray};
12+
use arrow::datatypes::{DataType, Field, FieldRef};
13+
use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};
14+
use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
15+
use std::collections::HashMap;
16+
use std::fmt::Write;
17+
use std::sync::Arc;
18+
19+
/// An enumerated type: a static, ordered set of text labels, like
20+
/// `CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')` in PostgreSQL.
21+
///
22+
/// Values are stored as [`DataType::Int16`] codes, where a code is the position of the label
23+
/// in the declaration. Comparing and sorting codes therefore follows the declaration order of
24+
/// the labels, as in PostgreSQL.
25+
///
26+
/// The type is identified by the `datafusion.enum` extension type name. Its metadata is a JSON
27+
/// object with the type name and the labels, for example
28+
/// `{"name":"mood","labels":["sad","ok","happy"]}`. As the metadata is part of the Arrow schema,
29+
/// file formats that store the Arrow schema (such as Parquet and Arrow IPC) keep the type
30+
/// definition next to the data.
31+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
32+
pub struct DFEnum {
33+
name: String,
34+
labels: Vec<String>,
35+
}
36+
37+
impl DFEnum {
38+
/// The extension type name of enumerated types.
39+
pub const NAME: &'static str = "datafusion.enum";
40+
41+
/// The storage type of the codes.
42+
pub const STORAGE_TYPE: DataType = DataType::Int16;
43+
44+
/// Creates a new enumerated type named `name` with the `labels` in declaration order.
45+
pub fn try_new(name: impl Into<String>, labels: Vec<String>) -> Result<Self> {
46+
let name = name.into();
47+
if labels.len() > i16::MAX as usize + 1 {
48+
return _plan_err!(
49+
"Enum type {name} has {} labels, at most {} are supported",
50+
labels.len(),
51+
i16::MAX as usize + 1
52+
);
53+
}
54+
let mut seen = std::collections::HashSet::with_capacity(labels.len());
55+
for label in &labels {
56+
if !seen.insert(label.as_str()) {
57+
return _plan_err!("Enum label \"{label}\" used more than once");
58+
}
59+
}
60+
Ok(Self { name, labels })
61+
}
62+
63+
/// Creates an enumerated type from the storage type and the serialized metadata of a field.
64+
pub fn try_from_metadata(
65+
data_type: &DataType,
66+
metadata: Option<&str>,
67+
) -> Result<Self> {
68+
if data_type != &Self::STORAGE_TYPE {
69+
return _plan_err!(
70+
"Enum type requires storage type {}, got {data_type}",
71+
Self::STORAGE_TYPE
72+
);
73+
}
74+
let Some(metadata) = metadata else {
75+
return _plan_err!("Enum type requires metadata");
76+
};
77+
let invalid = || _plan_datafusion_err!("Invalid enum type metadata: {metadata}");
78+
let value: serde_json::Value =
79+
serde_json::from_str(metadata).map_err(|_| invalid())?;
80+
let name = value
81+
.get("name")
82+
.and_then(|name| name.as_str())
83+
.ok_or_else(invalid)?;
84+
let labels = value
85+
.get("labels")
86+
.and_then(|labels| labels.as_array())
87+
.ok_or_else(invalid)?
88+
.iter()
89+
.map(|label| label.as_str().map(str::to_owned).ok_or_else(invalid))
90+
.collect::<Result<Vec<_>>>()?;
91+
Self::try_new(name, labels)
92+
}
93+
94+
/// Returns the enumerated type of `field`, or `None` if the field is not an enum.
95+
pub fn try_from_field(field: &Field) -> Result<Option<Self>> {
96+
if field.extension_type_name() != Some(Self::NAME) {
97+
return Ok(None);
98+
}
99+
Self::try_from_metadata(field.data_type(), field.extension_type_metadata())
100+
.map(Some)
101+
}
102+
103+
/// The name of the type.
104+
pub fn name(&self) -> &str {
105+
&self.name
106+
}
107+
108+
/// The labels in declaration order.
109+
pub fn labels(&self) -> &[String] {
110+
&self.labels
111+
}
112+
113+
/// Returns the metadata to attach to a field of this type.
114+
pub fn field_metadata(&self) -> HashMap<String, String> {
115+
HashMap::from([
116+
(EXTENSION_TYPE_NAME_KEY.to_owned(), Self::NAME.to_owned()),
117+
(
118+
EXTENSION_TYPE_METADATA_KEY.to_owned(),
119+
self.serialize_metadata_json(),
120+
),
121+
])
122+
}
123+
124+
/// Returns a nullable, unnamed field of this type.
125+
pub fn to_field_ref(&self) -> FieldRef {
126+
Arc::new(
127+
Self::STORAGE_TYPE
128+
.into_nullable_field()
129+
.with_metadata(self.field_metadata()),
130+
)
131+
}
132+
133+
/// Returns the code of `label`.
134+
pub fn code(&self, label: &str) -> Result<i16> {
135+
self.labels
136+
.iter()
137+
.position(|l| l == label)
138+
.map(|code| code as i16)
139+
.ok_or_else(|| {
140+
_plan_datafusion_err!(
141+
"invalid input value for enum {}: \"{label}\"",
142+
self.name
143+
)
144+
})
145+
}
146+
147+
/// Returns the label of `code`.
148+
pub fn label(&self, code: i16) -> Result<&str> {
149+
usize::try_from(code)
150+
.ok()
151+
.and_then(|code| self.labels.get(code))
152+
.map(String::as_str)
153+
.ok_or_else(|| {
154+
_plan_datafusion_err!("invalid code for enum {}: {code}", self.name)
155+
})
156+
}
157+
158+
/// Converts an array of labels (`Utf8`, `LargeUtf8` or `Utf8View`) to an array of codes.
159+
pub fn encode(&self, array: &dyn Array) -> Result<Int16Array> {
160+
let codes: HashMap<&str, i16> = self
161+
.labels
162+
.iter()
163+
.enumerate()
164+
.map(|(code, label)| (label.as_str(), code as i16))
165+
.collect();
166+
let encode = |label: Option<&str>| {
167+
label
168+
.map(|label| {
169+
codes.get(label).copied().ok_or_else(|| {
170+
_plan_datafusion_err!(
171+
"invalid input value for enum {}: \"{label}\"",
172+
self.name
173+
)
174+
})
175+
})
176+
.transpose()
177+
};
178+
match array.data_type() {
179+
DataType::Utf8 => array.as_string::<i32>().iter().map(encode).collect(),
180+
DataType::LargeUtf8 => array.as_string::<i64>().iter().map(encode).collect(),
181+
DataType::Utf8View => array.as_string_view().iter().map(encode).collect(),
182+
other => _internal_err!("Cannot convert {other} to enum {}", self.name),
183+
}
184+
}
185+
186+
/// Converts an array of codes to an array of labels.
187+
pub fn decode(&self, array: &Int16Array) -> Result<StringArray> {
188+
array
189+
.iter()
190+
.map(|code| code.map(|code| self.label(code)).transpose())
191+
.collect()
192+
}
193+
194+
fn serialize_metadata_json(&self) -> String {
195+
serde_json::json!({ "name": self.name, "labels": self.labels }).to_string()
196+
}
197+
}
198+
199+
impl DFExtensionType for DFEnum {
200+
fn storage_type(&self) -> DataType {
201+
Self::STORAGE_TYPE
202+
}
203+
204+
fn serialize_metadata(&self) -> Option<String> {
205+
Some(self.serialize_metadata_json())
206+
}
207+
208+
fn create_array_formatter<'fmt>(
209+
&self,
210+
array: &'fmt dyn Array,
211+
options: &FormatOptions<'fmt>,
212+
) -> Result<Option<ArrayFormatter<'fmt>>> {
213+
let Some(codes) = array.as_primitive_opt() else {
214+
return _internal_err!("Wrong array type for enum {}", self.name);
215+
};
216+
let display_index = EnumValueDisplayIndex {
217+
codes,
218+
labels: self.labels.clone(),
219+
null_str: options.null(),
220+
};
221+
Ok(Some(ArrayFormatter::new(
222+
Box::new(display_index),
223+
options.safe(),
224+
)))
225+
}
226+
}
227+
228+
/// Pretty printer for enum codes.
229+
struct EnumValueDisplayIndex<'a> {
230+
codes: &'a Int16Array,
231+
labels: Vec<String>,
232+
null_str: &'a str,
233+
}
234+
235+
impl DisplayIndex for EnumValueDisplayIndex<'_> {
236+
fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult {
237+
if self.codes.is_null(idx) {
238+
write!(f, "{}", self.null_str)?;
239+
return Ok(());
240+
}
241+
let code = self.codes.value(idx);
242+
match usize::try_from(code).ok().and_then(|c| self.labels.get(c)) {
243+
Some(label) => write!(f, "{label}")?,
244+
None => write!(f, "<invalid enum code {code}>")?,
245+
}
246+
Ok(())
247+
}
248+
}
249+
250+
#[cfg(test)]
251+
mod tests {
252+
use super::*;
253+
254+
fn mood() -> DFEnum {
255+
DFEnum::try_new("mood", vec!["sad".into(), "ok".into(), "happy".into()]).unwrap()
256+
}
257+
258+
#[test]
259+
fn metadata_roundtrip() -> Result<()> {
260+
let mood = mood();
261+
let field = mood.to_field_ref();
262+
assert_eq!(DFEnum::try_from_field(&field)?, Some(mood));
263+
assert_eq!(
264+
DFEnum::try_from_field(&Field::new("", DataType::Int16, true))?,
265+
None
266+
);
267+
Ok(())
268+
}
269+
270+
#[test]
271+
fn rejects_duplicate_labels() {
272+
let err = DFEnum::try_new("e", vec!["a".into(), "a".into()]).unwrap_err();
273+
assert!(err.to_string().contains("used more than once"), "{err}");
274+
}
275+
276+
#[test]
277+
fn encode_decode() -> Result<()> {
278+
let mood = mood();
279+
let labels = StringArray::from(vec![Some("happy"), None, Some("sad")]);
280+
let codes = mood.encode(&labels)?;
281+
assert_eq!(codes, Int16Array::from(vec![Some(2), None, Some(0)]));
282+
assert_eq!(mood.decode(&codes)?, labels);
283+
284+
let err = mood.encode(&StringArray::from(vec!["angry"])).unwrap_err();
285+
assert!(
286+
err.to_string()
287+
.contains("invalid input value for enum mood: \"angry\""),
288+
"{err}"
289+
);
290+
Ok(())
291+
}
292+
293+
#[test]
294+
fn pretty_print() -> Result<()> {
295+
let codes = Int16Array::from(vec![Some(1), None]);
296+
let formatter = mood()
297+
.create_array_formatter(&codes, &FormatOptions::default().with_null("NULL"))?
298+
.unwrap();
299+
assert_eq!(formatter.value(0).to_string(), "ok");
300+
assert_eq!(formatter.value(1).to_string(), "NULL");
301+
Ok(())
302+
}
303+
}

‎datafusion/common/src/types/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@
44

55
mod builtin;
66
mod canonical_extensions;
7+
mod enum_type;
78
mod extension;
89
mod field;
910
mod logical;
1011
mod native;
1112

1213
pub use builtin::*;
1314
pub use canonical_extensions::*;
15+
pub use enum_type::*;
1416
pub use extension::*;
1517
pub use field::*;
1618
pub use logical::*;

‎datafusion/core/src/execution/session_state_defaults.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,14 @@ use crate::functions_nested;
1919
use crate::{functions, functions_aggregate, functions_table, functions_window};
2020
use datafusion_catalog::TableFunction;
2121
use datafusion_catalog::{MemoryCatalogProvider, MemorySchemaProvider};
22+
use datafusion_common::types::DFEnum;
2223
use datafusion_execution::config::SessionConfig;
2324
use datafusion_execution::object_store::ObjectStoreUrl;
2425
use datafusion_execution::runtime_env::RuntimeEnv;
2526
use datafusion_expr::planner::ExprPlanner;
26-
use datafusion_expr::registry::ExtensionTypeRegistrationRef;
27+
use datafusion_expr::registry::{
28+
ExtensionTypeRegistration, ExtensionTypeRegistrationRef,
29+
};
2730
use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF};
2831
use std::collections::HashMap;
2932
use std::sync::Arc;
@@ -123,7 +126,12 @@ impl SessionStateDefaults {
123126
///
124127
/// For now, we do not register any extension types by default.
125128
pub fn default_extension_types() -> Vec<ExtensionTypeRegistrationRef> {
126-
vec![]
129+
vec![ExtensionTypeRegistration::new_arc(
130+
DFEnum::NAME,
131+
|storage_type, metadata| {
132+
Ok(Arc::new(DFEnum::try_from_metadata(storage_type, metadata)?))
133+
},
134+
)]
127135
}
128136

129137
/// returns the list of default [`TableFunction`]s

0 commit comments

Comments
 (0)